perf_event.c 80 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139
  1. #undef DEBUG
  2. /*
  3. * ARM performance counter support.
  4. *
  5. * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles
  6. *
  7. * ARMv7 support: Jean Pihet <jpihet@mvista.com>
  8. * 2010 (c) MontaVista Software, LLC.
  9. *
  10. * This code is based on the sparc64 perf event code, which is in turn based
  11. * on the x86 code. Callchain code is based on the ARM OProfile backtrace
  12. * code.
  13. */
  14. #define pr_fmt(fmt) "hw perfevents: " fmt
  15. #include <linux/interrupt.h>
  16. #include <linux/kernel.h>
  17. #include <linux/module.h>
  18. #include <linux/perf_event.h>
  19. #include <linux/platform_device.h>
  20. #include <linux/spinlock.h>
  21. #include <linux/uaccess.h>
  22. #include <asm/cputype.h>
  23. #include <asm/irq.h>
  24. #include <asm/irq_regs.h>
  25. #include <asm/pmu.h>
  26. #include <asm/stacktrace.h>
  27. static struct platform_device *pmu_device;
  28. /*
  29. * Hardware lock to serialize accesses to PMU registers. Needed for the
  30. * read/modify/write sequences.
  31. */
  32. DEFINE_SPINLOCK(pmu_lock);
  33. /*
  34. * ARMv6 supports a maximum of 3 events, starting from index 1. If we add
  35. * another platform that supports more, we need to increase this to be the
  36. * largest of all platforms.
  37. *
  38. * ARMv7 supports up to 32 events:
  39. * cycle counter CCNT + 31 events counters CNT0..30.
  40. * Cortex-A8 has 1+4 counters, Cortex-A9 has 1+6 counters.
  41. */
  42. #define ARMPMU_MAX_HWEVENTS 33
  43. /* The events for a given CPU. */
  44. struct cpu_hw_events {
  45. /*
  46. * The events that are active on the CPU for the given index. Index 0
  47. * is reserved.
  48. */
  49. struct perf_event *events[ARMPMU_MAX_HWEVENTS];
  50. /*
  51. * A 1 bit for an index indicates that the counter is being used for
  52. * an event. A 0 means that the counter can be used.
  53. */
  54. unsigned long used_mask[BITS_TO_LONGS(ARMPMU_MAX_HWEVENTS)];
  55. /*
  56. * A 1 bit for an index indicates that the counter is actively being
  57. * used.
  58. */
  59. unsigned long active_mask[BITS_TO_LONGS(ARMPMU_MAX_HWEVENTS)];
  60. };
  61. DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
  62. /* PMU names. */
  63. static const char *arm_pmu_names[] = {
  64. [ARM_PERF_PMU_ID_XSCALE1] = "xscale1",
  65. [ARM_PERF_PMU_ID_XSCALE2] = "xscale2",
  66. [ARM_PERF_PMU_ID_V6] = "v6",
  67. [ARM_PERF_PMU_ID_V6MP] = "v6mpcore",
  68. [ARM_PERF_PMU_ID_CA8] = "ARMv7 Cortex-A8",
  69. [ARM_PERF_PMU_ID_CA9] = "ARMv7 Cortex-A9",
  70. };
  71. struct arm_pmu {
  72. enum arm_perf_pmu_ids id;
  73. irqreturn_t (*handle_irq)(int irq_num, void *dev);
  74. void (*enable)(struct hw_perf_event *evt, int idx);
  75. void (*disable)(struct hw_perf_event *evt, int idx);
  76. int (*event_map)(int evt);
  77. u64 (*raw_event)(u64);
  78. int (*get_event_idx)(struct cpu_hw_events *cpuc,
  79. struct hw_perf_event *hwc);
  80. u32 (*read_counter)(int idx);
  81. void (*write_counter)(int idx, u32 val);
  82. void (*start)(void);
  83. void (*stop)(void);
  84. int num_events;
  85. u64 max_period;
  86. };
  87. /* Set at runtime when we know what CPU type we are. */
  88. static const struct arm_pmu *armpmu;
  89. enum arm_perf_pmu_ids
  90. armpmu_get_pmu_id(void)
  91. {
  92. int id = -ENODEV;
  93. if (armpmu != NULL)
  94. id = armpmu->id;
  95. return id;
  96. }
  97. EXPORT_SYMBOL_GPL(armpmu_get_pmu_id);
  98. int
  99. armpmu_get_max_events(void)
  100. {
  101. int max_events = 0;
  102. if (armpmu != NULL)
  103. max_events = armpmu->num_events;
  104. return max_events;
  105. }
  106. EXPORT_SYMBOL_GPL(armpmu_get_max_events);
  107. int perf_num_counters(void)
  108. {
  109. return armpmu_get_max_events();
  110. }
  111. EXPORT_SYMBOL_GPL(perf_num_counters);
  112. #define HW_OP_UNSUPPORTED 0xFFFF
  113. #define C(_x) \
  114. PERF_COUNT_HW_CACHE_##_x
  115. #define CACHE_OP_UNSUPPORTED 0xFFFF
  116. static unsigned armpmu_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  117. [PERF_COUNT_HW_CACHE_OP_MAX]
  118. [PERF_COUNT_HW_CACHE_RESULT_MAX];
  119. static int
  120. armpmu_map_cache_event(u64 config)
  121. {
  122. unsigned int cache_type, cache_op, cache_result, ret;
  123. cache_type = (config >> 0) & 0xff;
  124. if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
  125. return -EINVAL;
  126. cache_op = (config >> 8) & 0xff;
  127. if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
  128. return -EINVAL;
  129. cache_result = (config >> 16) & 0xff;
  130. if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
  131. return -EINVAL;
  132. ret = (int)armpmu_perf_cache_map[cache_type][cache_op][cache_result];
  133. if (ret == CACHE_OP_UNSUPPORTED)
  134. return -ENOENT;
  135. return ret;
  136. }
  137. static int
  138. armpmu_event_set_period(struct perf_event *event,
  139. struct hw_perf_event *hwc,
  140. int idx)
  141. {
  142. s64 left = local64_read(&hwc->period_left);
  143. s64 period = hwc->sample_period;
  144. int ret = 0;
  145. if (unlikely(left <= -period)) {
  146. left = period;
  147. local64_set(&hwc->period_left, left);
  148. hwc->last_period = period;
  149. ret = 1;
  150. }
  151. if (unlikely(left <= 0)) {
  152. left += period;
  153. local64_set(&hwc->period_left, left);
  154. hwc->last_period = period;
  155. ret = 1;
  156. }
  157. if (left > (s64)armpmu->max_period)
  158. left = armpmu->max_period;
  159. local64_set(&hwc->prev_count, (u64)-left);
  160. armpmu->write_counter(idx, (u64)(-left) & 0xffffffff);
  161. perf_event_update_userpage(event);
  162. return ret;
  163. }
  164. static u64
  165. armpmu_event_update(struct perf_event *event,
  166. struct hw_perf_event *hwc,
  167. int idx)
  168. {
  169. int shift = 64 - 32;
  170. s64 prev_raw_count, new_raw_count;
  171. u64 delta;
  172. again:
  173. prev_raw_count = local64_read(&hwc->prev_count);
  174. new_raw_count = armpmu->read_counter(idx);
  175. if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
  176. new_raw_count) != prev_raw_count)
  177. goto again;
  178. delta = (new_raw_count << shift) - (prev_raw_count << shift);
  179. delta >>= shift;
  180. local64_add(delta, &event->count);
  181. local64_sub(delta, &hwc->period_left);
  182. return new_raw_count;
  183. }
  184. static void
  185. armpmu_disable(struct perf_event *event)
  186. {
  187. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  188. struct hw_perf_event *hwc = &event->hw;
  189. int idx = hwc->idx;
  190. WARN_ON(idx < 0);
  191. clear_bit(idx, cpuc->active_mask);
  192. armpmu->disable(hwc, idx);
  193. barrier();
  194. armpmu_event_update(event, hwc, idx);
  195. cpuc->events[idx] = NULL;
  196. clear_bit(idx, cpuc->used_mask);
  197. perf_event_update_userpage(event);
  198. }
  199. static void
  200. armpmu_read(struct perf_event *event)
  201. {
  202. struct hw_perf_event *hwc = &event->hw;
  203. /* Don't read disabled counters! */
  204. if (hwc->idx < 0)
  205. return;
  206. armpmu_event_update(event, hwc, hwc->idx);
  207. }
  208. static void
  209. armpmu_unthrottle(struct perf_event *event)
  210. {
  211. struct hw_perf_event *hwc = &event->hw;
  212. /*
  213. * Set the period again. Some counters can't be stopped, so when we
  214. * were throttled we simply disabled the IRQ source and the counter
  215. * may have been left counting. If we don't do this step then we may
  216. * get an interrupt too soon or *way* too late if the overflow has
  217. * happened since disabling.
  218. */
  219. armpmu_event_set_period(event, hwc, hwc->idx);
  220. armpmu->enable(hwc, hwc->idx);
  221. }
  222. static int
  223. armpmu_enable(struct perf_event *event)
  224. {
  225. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  226. struct hw_perf_event *hwc = &event->hw;
  227. int idx;
  228. int err = 0;
  229. /* If we don't have a space for the counter then finish early. */
  230. idx = armpmu->get_event_idx(cpuc, hwc);
  231. if (idx < 0) {
  232. err = idx;
  233. goto out;
  234. }
  235. /*
  236. * If there is an event in the counter we are going to use then make
  237. * sure it is disabled.
  238. */
  239. event->hw.idx = idx;
  240. armpmu->disable(hwc, idx);
  241. cpuc->events[idx] = event;
  242. set_bit(idx, cpuc->active_mask);
  243. /* Set the period for the event. */
  244. armpmu_event_set_period(event, hwc, idx);
  245. /* Enable the event. */
  246. armpmu->enable(hwc, idx);
  247. /* Propagate our changes to the userspace mapping. */
  248. perf_event_update_userpage(event);
  249. out:
  250. return err;
  251. }
  252. static struct pmu pmu = {
  253. .enable = armpmu_enable,
  254. .disable = armpmu_disable,
  255. .unthrottle = armpmu_unthrottle,
  256. .read = armpmu_read,
  257. };
  258. static int
  259. validate_event(struct cpu_hw_events *cpuc,
  260. struct perf_event *event)
  261. {
  262. struct hw_perf_event fake_event = event->hw;
  263. if (event->pmu != &pmu || event->state <= PERF_EVENT_STATE_OFF)
  264. return 1;
  265. return armpmu->get_event_idx(cpuc, &fake_event) >= 0;
  266. }
  267. static int
  268. validate_group(struct perf_event *event)
  269. {
  270. struct perf_event *sibling, *leader = event->group_leader;
  271. struct cpu_hw_events fake_pmu;
  272. memset(&fake_pmu, 0, sizeof(fake_pmu));
  273. if (!validate_event(&fake_pmu, leader))
  274. return -ENOSPC;
  275. list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
  276. if (!validate_event(&fake_pmu, sibling))
  277. return -ENOSPC;
  278. }
  279. if (!validate_event(&fake_pmu, event))
  280. return -ENOSPC;
  281. return 0;
  282. }
  283. static int
  284. armpmu_reserve_hardware(void)
  285. {
  286. int i, err = -ENODEV, irq;
  287. pmu_device = reserve_pmu(ARM_PMU_DEVICE_CPU);
  288. if (IS_ERR(pmu_device)) {
  289. pr_warning("unable to reserve pmu\n");
  290. return PTR_ERR(pmu_device);
  291. }
  292. init_pmu(ARM_PMU_DEVICE_CPU);
  293. if (pmu_device->num_resources < 1) {
  294. pr_err("no irqs for PMUs defined\n");
  295. return -ENODEV;
  296. }
  297. for (i = 0; i < pmu_device->num_resources; ++i) {
  298. irq = platform_get_irq(pmu_device, i);
  299. if (irq < 0)
  300. continue;
  301. err = request_irq(irq, armpmu->handle_irq,
  302. IRQF_DISABLED | IRQF_NOBALANCING,
  303. "armpmu", NULL);
  304. if (err) {
  305. pr_warning("unable to request IRQ%d for ARM perf "
  306. "counters\n", irq);
  307. break;
  308. }
  309. }
  310. if (err) {
  311. for (i = i - 1; i >= 0; --i) {
  312. irq = platform_get_irq(pmu_device, i);
  313. if (irq >= 0)
  314. free_irq(irq, NULL);
  315. }
  316. release_pmu(pmu_device);
  317. pmu_device = NULL;
  318. }
  319. return err;
  320. }
  321. static void
  322. armpmu_release_hardware(void)
  323. {
  324. int i, irq;
  325. for (i = pmu_device->num_resources - 1; i >= 0; --i) {
  326. irq = platform_get_irq(pmu_device, i);
  327. if (irq >= 0)
  328. free_irq(irq, NULL);
  329. }
  330. armpmu->stop();
  331. release_pmu(pmu_device);
  332. pmu_device = NULL;
  333. }
  334. static atomic_t active_events = ATOMIC_INIT(0);
  335. static DEFINE_MUTEX(pmu_reserve_mutex);
  336. static void
  337. hw_perf_event_destroy(struct perf_event *event)
  338. {
  339. if (atomic_dec_and_mutex_lock(&active_events, &pmu_reserve_mutex)) {
  340. armpmu_release_hardware();
  341. mutex_unlock(&pmu_reserve_mutex);
  342. }
  343. }
  344. static int
  345. __hw_perf_event_init(struct perf_event *event)
  346. {
  347. struct hw_perf_event *hwc = &event->hw;
  348. int mapping, err;
  349. /* Decode the generic type into an ARM event identifier. */
  350. if (PERF_TYPE_HARDWARE == event->attr.type) {
  351. mapping = armpmu->event_map(event->attr.config);
  352. } else if (PERF_TYPE_HW_CACHE == event->attr.type) {
  353. mapping = armpmu_map_cache_event(event->attr.config);
  354. } else if (PERF_TYPE_RAW == event->attr.type) {
  355. mapping = armpmu->raw_event(event->attr.config);
  356. } else {
  357. pr_debug("event type %x not supported\n", event->attr.type);
  358. return -EOPNOTSUPP;
  359. }
  360. if (mapping < 0) {
  361. pr_debug("event %x:%llx not supported\n", event->attr.type,
  362. event->attr.config);
  363. return mapping;
  364. }
  365. /*
  366. * Check whether we need to exclude the counter from certain modes.
  367. * The ARM performance counters are on all of the time so if someone
  368. * has asked us for some excludes then we have to fail.
  369. */
  370. if (event->attr.exclude_kernel || event->attr.exclude_user ||
  371. event->attr.exclude_hv || event->attr.exclude_idle) {
  372. pr_debug("ARM performance counters do not support "
  373. "mode exclusion\n");
  374. return -EPERM;
  375. }
  376. /*
  377. * We don't assign an index until we actually place the event onto
  378. * hardware. Use -1 to signify that we haven't decided where to put it
  379. * yet. For SMP systems, each core has it's own PMU so we can't do any
  380. * clever allocation or constraints checking at this point.
  381. */
  382. hwc->idx = -1;
  383. /*
  384. * Store the event encoding into the config_base field. config and
  385. * event_base are unused as the only 2 things we need to know are
  386. * the event mapping and the counter to use. The counter to use is
  387. * also the indx and the config_base is the event type.
  388. */
  389. hwc->config_base = (unsigned long)mapping;
  390. hwc->config = 0;
  391. hwc->event_base = 0;
  392. if (!hwc->sample_period) {
  393. hwc->sample_period = armpmu->max_period;
  394. hwc->last_period = hwc->sample_period;
  395. local64_set(&hwc->period_left, hwc->sample_period);
  396. }
  397. err = 0;
  398. if (event->group_leader != event) {
  399. err = validate_group(event);
  400. if (err)
  401. return -EINVAL;
  402. }
  403. return err;
  404. }
  405. const struct pmu *
  406. hw_perf_event_init(struct perf_event *event)
  407. {
  408. int err = 0;
  409. if (!armpmu)
  410. return ERR_PTR(-ENODEV);
  411. event->destroy = hw_perf_event_destroy;
  412. if (!atomic_inc_not_zero(&active_events)) {
  413. if (atomic_read(&active_events) > perf_max_events) {
  414. atomic_dec(&active_events);
  415. return ERR_PTR(-ENOSPC);
  416. }
  417. mutex_lock(&pmu_reserve_mutex);
  418. if (atomic_read(&active_events) == 0) {
  419. err = armpmu_reserve_hardware();
  420. }
  421. if (!err)
  422. atomic_inc(&active_events);
  423. mutex_unlock(&pmu_reserve_mutex);
  424. }
  425. if (err)
  426. return ERR_PTR(err);
  427. err = __hw_perf_event_init(event);
  428. if (err)
  429. hw_perf_event_destroy(event);
  430. return err ? ERR_PTR(err) : &pmu;
  431. }
  432. void
  433. hw_perf_enable(void)
  434. {
  435. /* Enable all of the perf events on hardware. */
  436. int idx;
  437. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  438. if (!armpmu)
  439. return;
  440. for (idx = 0; idx <= armpmu->num_events; ++idx) {
  441. struct perf_event *event = cpuc->events[idx];
  442. if (!event)
  443. continue;
  444. armpmu->enable(&event->hw, idx);
  445. }
  446. armpmu->start();
  447. }
  448. void
  449. hw_perf_disable(void)
  450. {
  451. if (armpmu)
  452. armpmu->stop();
  453. }
  454. /*
  455. * ARMv6 Performance counter handling code.
  456. *
  457. * ARMv6 has 2 configurable performance counters and a single cycle counter.
  458. * They all share a single reset bit but can be written to zero so we can use
  459. * that for a reset.
  460. *
  461. * The counters can't be individually enabled or disabled so when we remove
  462. * one event and replace it with another we could get spurious counts from the
  463. * wrong event. However, we can take advantage of the fact that the
  464. * performance counters can export events to the event bus, and the event bus
  465. * itself can be monitored. This requires that we *don't* export the events to
  466. * the event bus. The procedure for disabling a configurable counter is:
  467. * - change the counter to count the ETMEXTOUT[0] signal (0x20). This
  468. * effectively stops the counter from counting.
  469. * - disable the counter's interrupt generation (each counter has it's
  470. * own interrupt enable bit).
  471. * Once stopped, the counter value can be written as 0 to reset.
  472. *
  473. * To enable a counter:
  474. * - enable the counter's interrupt generation.
  475. * - set the new event type.
  476. *
  477. * Note: the dedicated cycle counter only counts cycles and can't be
  478. * enabled/disabled independently of the others. When we want to disable the
  479. * cycle counter, we have to just disable the interrupt reporting and start
  480. * ignoring that counter. When re-enabling, we have to reset the value and
  481. * enable the interrupt.
  482. */
  483. enum armv6_perf_types {
  484. ARMV6_PERFCTR_ICACHE_MISS = 0x0,
  485. ARMV6_PERFCTR_IBUF_STALL = 0x1,
  486. ARMV6_PERFCTR_DDEP_STALL = 0x2,
  487. ARMV6_PERFCTR_ITLB_MISS = 0x3,
  488. ARMV6_PERFCTR_DTLB_MISS = 0x4,
  489. ARMV6_PERFCTR_BR_EXEC = 0x5,
  490. ARMV6_PERFCTR_BR_MISPREDICT = 0x6,
  491. ARMV6_PERFCTR_INSTR_EXEC = 0x7,
  492. ARMV6_PERFCTR_DCACHE_HIT = 0x9,
  493. ARMV6_PERFCTR_DCACHE_ACCESS = 0xA,
  494. ARMV6_PERFCTR_DCACHE_MISS = 0xB,
  495. ARMV6_PERFCTR_DCACHE_WBACK = 0xC,
  496. ARMV6_PERFCTR_SW_PC_CHANGE = 0xD,
  497. ARMV6_PERFCTR_MAIN_TLB_MISS = 0xF,
  498. ARMV6_PERFCTR_EXPL_D_ACCESS = 0x10,
  499. ARMV6_PERFCTR_LSU_FULL_STALL = 0x11,
  500. ARMV6_PERFCTR_WBUF_DRAINED = 0x12,
  501. ARMV6_PERFCTR_CPU_CYCLES = 0xFF,
  502. ARMV6_PERFCTR_NOP = 0x20,
  503. };
  504. enum armv6_counters {
  505. ARMV6_CYCLE_COUNTER = 1,
  506. ARMV6_COUNTER0,
  507. ARMV6_COUNTER1,
  508. };
  509. /*
  510. * The hardware events that we support. We do support cache operations but
  511. * we have harvard caches and no way to combine instruction and data
  512. * accesses/misses in hardware.
  513. */
  514. static const unsigned armv6_perf_map[PERF_COUNT_HW_MAX] = {
  515. [PERF_COUNT_HW_CPU_CYCLES] = ARMV6_PERFCTR_CPU_CYCLES,
  516. [PERF_COUNT_HW_INSTRUCTIONS] = ARMV6_PERFCTR_INSTR_EXEC,
  517. [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED,
  518. [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED,
  519. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6_PERFCTR_BR_EXEC,
  520. [PERF_COUNT_HW_BRANCH_MISSES] = ARMV6_PERFCTR_BR_MISPREDICT,
  521. [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED,
  522. };
  523. static const unsigned armv6_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  524. [PERF_COUNT_HW_CACHE_OP_MAX]
  525. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  526. [C(L1D)] = {
  527. /*
  528. * The performance counters don't differentiate between read
  529. * and write accesses/misses so this isn't strictly correct,
  530. * but it's the best we can do. Writes and reads get
  531. * combined.
  532. */
  533. [C(OP_READ)] = {
  534. [C(RESULT_ACCESS)] = ARMV6_PERFCTR_DCACHE_ACCESS,
  535. [C(RESULT_MISS)] = ARMV6_PERFCTR_DCACHE_MISS,
  536. },
  537. [C(OP_WRITE)] = {
  538. [C(RESULT_ACCESS)] = ARMV6_PERFCTR_DCACHE_ACCESS,
  539. [C(RESULT_MISS)] = ARMV6_PERFCTR_DCACHE_MISS,
  540. },
  541. [C(OP_PREFETCH)] = {
  542. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  543. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  544. },
  545. },
  546. [C(L1I)] = {
  547. [C(OP_READ)] = {
  548. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  549. [C(RESULT_MISS)] = ARMV6_PERFCTR_ICACHE_MISS,
  550. },
  551. [C(OP_WRITE)] = {
  552. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  553. [C(RESULT_MISS)] = ARMV6_PERFCTR_ICACHE_MISS,
  554. },
  555. [C(OP_PREFETCH)] = {
  556. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  557. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  558. },
  559. },
  560. [C(LL)] = {
  561. [C(OP_READ)] = {
  562. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  563. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  564. },
  565. [C(OP_WRITE)] = {
  566. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  567. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  568. },
  569. [C(OP_PREFETCH)] = {
  570. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  571. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  572. },
  573. },
  574. [C(DTLB)] = {
  575. /*
  576. * The ARM performance counters can count micro DTLB misses,
  577. * micro ITLB misses and main TLB misses. There isn't an event
  578. * for TLB misses, so use the micro misses here and if users
  579. * want the main TLB misses they can use a raw counter.
  580. */
  581. [C(OP_READ)] = {
  582. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  583. [C(RESULT_MISS)] = ARMV6_PERFCTR_DTLB_MISS,
  584. },
  585. [C(OP_WRITE)] = {
  586. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  587. [C(RESULT_MISS)] = ARMV6_PERFCTR_DTLB_MISS,
  588. },
  589. [C(OP_PREFETCH)] = {
  590. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  591. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  592. },
  593. },
  594. [C(ITLB)] = {
  595. [C(OP_READ)] = {
  596. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  597. [C(RESULT_MISS)] = ARMV6_PERFCTR_ITLB_MISS,
  598. },
  599. [C(OP_WRITE)] = {
  600. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  601. [C(RESULT_MISS)] = ARMV6_PERFCTR_ITLB_MISS,
  602. },
  603. [C(OP_PREFETCH)] = {
  604. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  605. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  606. },
  607. },
  608. [C(BPU)] = {
  609. [C(OP_READ)] = {
  610. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  611. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  612. },
  613. [C(OP_WRITE)] = {
  614. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  615. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  616. },
  617. [C(OP_PREFETCH)] = {
  618. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  619. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  620. },
  621. },
  622. };
  623. enum armv6mpcore_perf_types {
  624. ARMV6MPCORE_PERFCTR_ICACHE_MISS = 0x0,
  625. ARMV6MPCORE_PERFCTR_IBUF_STALL = 0x1,
  626. ARMV6MPCORE_PERFCTR_DDEP_STALL = 0x2,
  627. ARMV6MPCORE_PERFCTR_ITLB_MISS = 0x3,
  628. ARMV6MPCORE_PERFCTR_DTLB_MISS = 0x4,
  629. ARMV6MPCORE_PERFCTR_BR_EXEC = 0x5,
  630. ARMV6MPCORE_PERFCTR_BR_NOTPREDICT = 0x6,
  631. ARMV6MPCORE_PERFCTR_BR_MISPREDICT = 0x7,
  632. ARMV6MPCORE_PERFCTR_INSTR_EXEC = 0x8,
  633. ARMV6MPCORE_PERFCTR_DCACHE_RDACCESS = 0xA,
  634. ARMV6MPCORE_PERFCTR_DCACHE_RDMISS = 0xB,
  635. ARMV6MPCORE_PERFCTR_DCACHE_WRACCESS = 0xC,
  636. ARMV6MPCORE_PERFCTR_DCACHE_WRMISS = 0xD,
  637. ARMV6MPCORE_PERFCTR_DCACHE_EVICTION = 0xE,
  638. ARMV6MPCORE_PERFCTR_SW_PC_CHANGE = 0xF,
  639. ARMV6MPCORE_PERFCTR_MAIN_TLB_MISS = 0x10,
  640. ARMV6MPCORE_PERFCTR_EXPL_MEM_ACCESS = 0x11,
  641. ARMV6MPCORE_PERFCTR_LSU_FULL_STALL = 0x12,
  642. ARMV6MPCORE_PERFCTR_WBUF_DRAINED = 0x13,
  643. ARMV6MPCORE_PERFCTR_CPU_CYCLES = 0xFF,
  644. };
  645. /*
  646. * The hardware events that we support. We do support cache operations but
  647. * we have harvard caches and no way to combine instruction and data
  648. * accesses/misses in hardware.
  649. */
  650. static const unsigned armv6mpcore_perf_map[PERF_COUNT_HW_MAX] = {
  651. [PERF_COUNT_HW_CPU_CYCLES] = ARMV6MPCORE_PERFCTR_CPU_CYCLES,
  652. [PERF_COUNT_HW_INSTRUCTIONS] = ARMV6MPCORE_PERFCTR_INSTR_EXEC,
  653. [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED,
  654. [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED,
  655. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6MPCORE_PERFCTR_BR_EXEC,
  656. [PERF_COUNT_HW_BRANCH_MISSES] = ARMV6MPCORE_PERFCTR_BR_MISPREDICT,
  657. [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED,
  658. };
  659. static const unsigned armv6mpcore_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  660. [PERF_COUNT_HW_CACHE_OP_MAX]
  661. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  662. [C(L1D)] = {
  663. [C(OP_READ)] = {
  664. [C(RESULT_ACCESS)] =
  665. ARMV6MPCORE_PERFCTR_DCACHE_RDACCESS,
  666. [C(RESULT_MISS)] =
  667. ARMV6MPCORE_PERFCTR_DCACHE_RDMISS,
  668. },
  669. [C(OP_WRITE)] = {
  670. [C(RESULT_ACCESS)] =
  671. ARMV6MPCORE_PERFCTR_DCACHE_WRACCESS,
  672. [C(RESULT_MISS)] =
  673. ARMV6MPCORE_PERFCTR_DCACHE_WRMISS,
  674. },
  675. [C(OP_PREFETCH)] = {
  676. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  677. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  678. },
  679. },
  680. [C(L1I)] = {
  681. [C(OP_READ)] = {
  682. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  683. [C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ICACHE_MISS,
  684. },
  685. [C(OP_WRITE)] = {
  686. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  687. [C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ICACHE_MISS,
  688. },
  689. [C(OP_PREFETCH)] = {
  690. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  691. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  692. },
  693. },
  694. [C(LL)] = {
  695. [C(OP_READ)] = {
  696. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  697. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  698. },
  699. [C(OP_WRITE)] = {
  700. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  701. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  702. },
  703. [C(OP_PREFETCH)] = {
  704. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  705. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  706. },
  707. },
  708. [C(DTLB)] = {
  709. /*
  710. * The ARM performance counters can count micro DTLB misses,
  711. * micro ITLB misses and main TLB misses. There isn't an event
  712. * for TLB misses, so use the micro misses here and if users
  713. * want the main TLB misses they can use a raw counter.
  714. */
  715. [C(OP_READ)] = {
  716. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  717. [C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_DTLB_MISS,
  718. },
  719. [C(OP_WRITE)] = {
  720. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  721. [C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_DTLB_MISS,
  722. },
  723. [C(OP_PREFETCH)] = {
  724. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  725. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  726. },
  727. },
  728. [C(ITLB)] = {
  729. [C(OP_READ)] = {
  730. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  731. [C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ITLB_MISS,
  732. },
  733. [C(OP_WRITE)] = {
  734. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  735. [C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ITLB_MISS,
  736. },
  737. [C(OP_PREFETCH)] = {
  738. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  739. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  740. },
  741. },
  742. [C(BPU)] = {
  743. [C(OP_READ)] = {
  744. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  745. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  746. },
  747. [C(OP_WRITE)] = {
  748. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  749. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  750. },
  751. [C(OP_PREFETCH)] = {
  752. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  753. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  754. },
  755. },
  756. };
  757. static inline unsigned long
  758. armv6_pmcr_read(void)
  759. {
  760. u32 val;
  761. asm volatile("mrc p15, 0, %0, c15, c12, 0" : "=r"(val));
  762. return val;
  763. }
  764. static inline void
  765. armv6_pmcr_write(unsigned long val)
  766. {
  767. asm volatile("mcr p15, 0, %0, c15, c12, 0" : : "r"(val));
  768. }
  769. #define ARMV6_PMCR_ENABLE (1 << 0)
  770. #define ARMV6_PMCR_CTR01_RESET (1 << 1)
  771. #define ARMV6_PMCR_CCOUNT_RESET (1 << 2)
  772. #define ARMV6_PMCR_CCOUNT_DIV (1 << 3)
  773. #define ARMV6_PMCR_COUNT0_IEN (1 << 4)
  774. #define ARMV6_PMCR_COUNT1_IEN (1 << 5)
  775. #define ARMV6_PMCR_CCOUNT_IEN (1 << 6)
  776. #define ARMV6_PMCR_COUNT0_OVERFLOW (1 << 8)
  777. #define ARMV6_PMCR_COUNT1_OVERFLOW (1 << 9)
  778. #define ARMV6_PMCR_CCOUNT_OVERFLOW (1 << 10)
  779. #define ARMV6_PMCR_EVT_COUNT0_SHIFT 20
  780. #define ARMV6_PMCR_EVT_COUNT0_MASK (0xFF << ARMV6_PMCR_EVT_COUNT0_SHIFT)
  781. #define ARMV6_PMCR_EVT_COUNT1_SHIFT 12
  782. #define ARMV6_PMCR_EVT_COUNT1_MASK (0xFF << ARMV6_PMCR_EVT_COUNT1_SHIFT)
  783. #define ARMV6_PMCR_OVERFLOWED_MASK \
  784. (ARMV6_PMCR_COUNT0_OVERFLOW | ARMV6_PMCR_COUNT1_OVERFLOW | \
  785. ARMV6_PMCR_CCOUNT_OVERFLOW)
  786. static inline int
  787. armv6_pmcr_has_overflowed(unsigned long pmcr)
  788. {
  789. return (pmcr & ARMV6_PMCR_OVERFLOWED_MASK);
  790. }
  791. static inline int
  792. armv6_pmcr_counter_has_overflowed(unsigned long pmcr,
  793. enum armv6_counters counter)
  794. {
  795. int ret = 0;
  796. if (ARMV6_CYCLE_COUNTER == counter)
  797. ret = pmcr & ARMV6_PMCR_CCOUNT_OVERFLOW;
  798. else if (ARMV6_COUNTER0 == counter)
  799. ret = pmcr & ARMV6_PMCR_COUNT0_OVERFLOW;
  800. else if (ARMV6_COUNTER1 == counter)
  801. ret = pmcr & ARMV6_PMCR_COUNT1_OVERFLOW;
  802. else
  803. WARN_ONCE(1, "invalid counter number (%d)\n", counter);
  804. return ret;
  805. }
  806. static inline u32
  807. armv6pmu_read_counter(int counter)
  808. {
  809. unsigned long value = 0;
  810. if (ARMV6_CYCLE_COUNTER == counter)
  811. asm volatile("mrc p15, 0, %0, c15, c12, 1" : "=r"(value));
  812. else if (ARMV6_COUNTER0 == counter)
  813. asm volatile("mrc p15, 0, %0, c15, c12, 2" : "=r"(value));
  814. else if (ARMV6_COUNTER1 == counter)
  815. asm volatile("mrc p15, 0, %0, c15, c12, 3" : "=r"(value));
  816. else
  817. WARN_ONCE(1, "invalid counter number (%d)\n", counter);
  818. return value;
  819. }
  820. static inline void
  821. armv6pmu_write_counter(int counter,
  822. u32 value)
  823. {
  824. if (ARMV6_CYCLE_COUNTER == counter)
  825. asm volatile("mcr p15, 0, %0, c15, c12, 1" : : "r"(value));
  826. else if (ARMV6_COUNTER0 == counter)
  827. asm volatile("mcr p15, 0, %0, c15, c12, 2" : : "r"(value));
  828. else if (ARMV6_COUNTER1 == counter)
  829. asm volatile("mcr p15, 0, %0, c15, c12, 3" : : "r"(value));
  830. else
  831. WARN_ONCE(1, "invalid counter number (%d)\n", counter);
  832. }
  833. void
  834. armv6pmu_enable_event(struct hw_perf_event *hwc,
  835. int idx)
  836. {
  837. unsigned long val, mask, evt, flags;
  838. if (ARMV6_CYCLE_COUNTER == idx) {
  839. mask = 0;
  840. evt = ARMV6_PMCR_CCOUNT_IEN;
  841. } else if (ARMV6_COUNTER0 == idx) {
  842. mask = ARMV6_PMCR_EVT_COUNT0_MASK;
  843. evt = (hwc->config_base << ARMV6_PMCR_EVT_COUNT0_SHIFT) |
  844. ARMV6_PMCR_COUNT0_IEN;
  845. } else if (ARMV6_COUNTER1 == idx) {
  846. mask = ARMV6_PMCR_EVT_COUNT1_MASK;
  847. evt = (hwc->config_base << ARMV6_PMCR_EVT_COUNT1_SHIFT) |
  848. ARMV6_PMCR_COUNT1_IEN;
  849. } else {
  850. WARN_ONCE(1, "invalid counter number (%d)\n", idx);
  851. return;
  852. }
  853. /*
  854. * Mask out the current event and set the counter to count the event
  855. * that we're interested in.
  856. */
  857. spin_lock_irqsave(&pmu_lock, flags);
  858. val = armv6_pmcr_read();
  859. val &= ~mask;
  860. val |= evt;
  861. armv6_pmcr_write(val);
  862. spin_unlock_irqrestore(&pmu_lock, flags);
  863. }
  864. static irqreturn_t
  865. armv6pmu_handle_irq(int irq_num,
  866. void *dev)
  867. {
  868. unsigned long pmcr = armv6_pmcr_read();
  869. struct perf_sample_data data;
  870. struct cpu_hw_events *cpuc;
  871. struct pt_regs *regs;
  872. int idx;
  873. if (!armv6_pmcr_has_overflowed(pmcr))
  874. return IRQ_NONE;
  875. regs = get_irq_regs();
  876. /*
  877. * The interrupts are cleared by writing the overflow flags back to
  878. * the control register. All of the other bits don't have any effect
  879. * if they are rewritten, so write the whole value back.
  880. */
  881. armv6_pmcr_write(pmcr);
  882. perf_sample_data_init(&data, 0);
  883. cpuc = &__get_cpu_var(cpu_hw_events);
  884. for (idx = 0; idx <= armpmu->num_events; ++idx) {
  885. struct perf_event *event = cpuc->events[idx];
  886. struct hw_perf_event *hwc;
  887. if (!test_bit(idx, cpuc->active_mask))
  888. continue;
  889. /*
  890. * We have a single interrupt for all counters. Check that
  891. * each counter has overflowed before we process it.
  892. */
  893. if (!armv6_pmcr_counter_has_overflowed(pmcr, idx))
  894. continue;
  895. hwc = &event->hw;
  896. armpmu_event_update(event, hwc, idx);
  897. data.period = event->hw.last_period;
  898. if (!armpmu_event_set_period(event, hwc, idx))
  899. continue;
  900. if (perf_event_overflow(event, 0, &data, regs))
  901. armpmu->disable(hwc, idx);
  902. }
  903. /*
  904. * Handle the pending perf events.
  905. *
  906. * Note: this call *must* be run with interrupts disabled. For
  907. * platforms that can have the PMU interrupts raised as an NMI, this
  908. * will not work.
  909. */
  910. perf_event_do_pending();
  911. return IRQ_HANDLED;
  912. }
  913. static void
  914. armv6pmu_start(void)
  915. {
  916. unsigned long flags, val;
  917. spin_lock_irqsave(&pmu_lock, flags);
  918. val = armv6_pmcr_read();
  919. val |= ARMV6_PMCR_ENABLE;
  920. armv6_pmcr_write(val);
  921. spin_unlock_irqrestore(&pmu_lock, flags);
  922. }
  923. void
  924. armv6pmu_stop(void)
  925. {
  926. unsigned long flags, val;
  927. spin_lock_irqsave(&pmu_lock, flags);
  928. val = armv6_pmcr_read();
  929. val &= ~ARMV6_PMCR_ENABLE;
  930. armv6_pmcr_write(val);
  931. spin_unlock_irqrestore(&pmu_lock, flags);
  932. }
  933. static inline int
  934. armv6pmu_event_map(int config)
  935. {
  936. int mapping = armv6_perf_map[config];
  937. if (HW_OP_UNSUPPORTED == mapping)
  938. mapping = -EOPNOTSUPP;
  939. return mapping;
  940. }
  941. static inline int
  942. armv6mpcore_pmu_event_map(int config)
  943. {
  944. int mapping = armv6mpcore_perf_map[config];
  945. if (HW_OP_UNSUPPORTED == mapping)
  946. mapping = -EOPNOTSUPP;
  947. return mapping;
  948. }
  949. static u64
  950. armv6pmu_raw_event(u64 config)
  951. {
  952. return config & 0xff;
  953. }
  954. static int
  955. armv6pmu_get_event_idx(struct cpu_hw_events *cpuc,
  956. struct hw_perf_event *event)
  957. {
  958. /* Always place a cycle counter into the cycle counter. */
  959. if (ARMV6_PERFCTR_CPU_CYCLES == event->config_base) {
  960. if (test_and_set_bit(ARMV6_CYCLE_COUNTER, cpuc->used_mask))
  961. return -EAGAIN;
  962. return ARMV6_CYCLE_COUNTER;
  963. } else {
  964. /*
  965. * For anything other than a cycle counter, try and use
  966. * counter0 and counter1.
  967. */
  968. if (!test_and_set_bit(ARMV6_COUNTER1, cpuc->used_mask)) {
  969. return ARMV6_COUNTER1;
  970. }
  971. if (!test_and_set_bit(ARMV6_COUNTER0, cpuc->used_mask)) {
  972. return ARMV6_COUNTER0;
  973. }
  974. /* The counters are all in use. */
  975. return -EAGAIN;
  976. }
  977. }
  978. static void
  979. armv6pmu_disable_event(struct hw_perf_event *hwc,
  980. int idx)
  981. {
  982. unsigned long val, mask, evt, flags;
  983. if (ARMV6_CYCLE_COUNTER == idx) {
  984. mask = ARMV6_PMCR_CCOUNT_IEN;
  985. evt = 0;
  986. } else if (ARMV6_COUNTER0 == idx) {
  987. mask = ARMV6_PMCR_COUNT0_IEN | ARMV6_PMCR_EVT_COUNT0_MASK;
  988. evt = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT0_SHIFT;
  989. } else if (ARMV6_COUNTER1 == idx) {
  990. mask = ARMV6_PMCR_COUNT1_IEN | ARMV6_PMCR_EVT_COUNT1_MASK;
  991. evt = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT1_SHIFT;
  992. } else {
  993. WARN_ONCE(1, "invalid counter number (%d)\n", idx);
  994. return;
  995. }
  996. /*
  997. * Mask out the current event and set the counter to count the number
  998. * of ETM bus signal assertion cycles. The external reporting should
  999. * be disabled and so this should never increment.
  1000. */
  1001. spin_lock_irqsave(&pmu_lock, flags);
  1002. val = armv6_pmcr_read();
  1003. val &= ~mask;
  1004. val |= evt;
  1005. armv6_pmcr_write(val);
  1006. spin_unlock_irqrestore(&pmu_lock, flags);
  1007. }
  1008. static void
  1009. armv6mpcore_pmu_disable_event(struct hw_perf_event *hwc,
  1010. int idx)
  1011. {
  1012. unsigned long val, mask, flags, evt = 0;
  1013. if (ARMV6_CYCLE_COUNTER == idx) {
  1014. mask = ARMV6_PMCR_CCOUNT_IEN;
  1015. } else if (ARMV6_COUNTER0 == idx) {
  1016. mask = ARMV6_PMCR_COUNT0_IEN;
  1017. } else if (ARMV6_COUNTER1 == idx) {
  1018. mask = ARMV6_PMCR_COUNT1_IEN;
  1019. } else {
  1020. WARN_ONCE(1, "invalid counter number (%d)\n", idx);
  1021. return;
  1022. }
  1023. /*
  1024. * Unlike UP ARMv6, we don't have a way of stopping the counters. We
  1025. * simply disable the interrupt reporting.
  1026. */
  1027. spin_lock_irqsave(&pmu_lock, flags);
  1028. val = armv6_pmcr_read();
  1029. val &= ~mask;
  1030. val |= evt;
  1031. armv6_pmcr_write(val);
  1032. spin_unlock_irqrestore(&pmu_lock, flags);
  1033. }
  1034. static const struct arm_pmu armv6pmu = {
  1035. .id = ARM_PERF_PMU_ID_V6,
  1036. .handle_irq = armv6pmu_handle_irq,
  1037. .enable = armv6pmu_enable_event,
  1038. .disable = armv6pmu_disable_event,
  1039. .event_map = armv6pmu_event_map,
  1040. .raw_event = armv6pmu_raw_event,
  1041. .read_counter = armv6pmu_read_counter,
  1042. .write_counter = armv6pmu_write_counter,
  1043. .get_event_idx = armv6pmu_get_event_idx,
  1044. .start = armv6pmu_start,
  1045. .stop = armv6pmu_stop,
  1046. .num_events = 3,
  1047. .max_period = (1LLU << 32) - 1,
  1048. };
  1049. /*
  1050. * ARMv6mpcore is almost identical to single core ARMv6 with the exception
  1051. * that some of the events have different enumerations and that there is no
  1052. * *hack* to stop the programmable counters. To stop the counters we simply
  1053. * disable the interrupt reporting and update the event. When unthrottling we
  1054. * reset the period and enable the interrupt reporting.
  1055. */
  1056. static const struct arm_pmu armv6mpcore_pmu = {
  1057. .id = ARM_PERF_PMU_ID_V6MP,
  1058. .handle_irq = armv6pmu_handle_irq,
  1059. .enable = armv6pmu_enable_event,
  1060. .disable = armv6mpcore_pmu_disable_event,
  1061. .event_map = armv6mpcore_pmu_event_map,
  1062. .raw_event = armv6pmu_raw_event,
  1063. .read_counter = armv6pmu_read_counter,
  1064. .write_counter = armv6pmu_write_counter,
  1065. .get_event_idx = armv6pmu_get_event_idx,
  1066. .start = armv6pmu_start,
  1067. .stop = armv6pmu_stop,
  1068. .num_events = 3,
  1069. .max_period = (1LLU << 32) - 1,
  1070. };
  1071. /*
  1072. * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
  1073. *
  1074. * Copied from ARMv6 code, with the low level code inspired
  1075. * by the ARMv7 Oprofile code.
  1076. *
  1077. * Cortex-A8 has up to 4 configurable performance counters and
  1078. * a single cycle counter.
  1079. * Cortex-A9 has up to 31 configurable performance counters and
  1080. * a single cycle counter.
  1081. *
  1082. * All counters can be enabled/disabled and IRQ masked separately. The cycle
  1083. * counter and all 4 performance counters together can be reset separately.
  1084. */
  1085. /* Common ARMv7 event types */
  1086. enum armv7_perf_types {
  1087. ARMV7_PERFCTR_PMNC_SW_INCR = 0x00,
  1088. ARMV7_PERFCTR_IFETCH_MISS = 0x01,
  1089. ARMV7_PERFCTR_ITLB_MISS = 0x02,
  1090. ARMV7_PERFCTR_DCACHE_REFILL = 0x03,
  1091. ARMV7_PERFCTR_DCACHE_ACCESS = 0x04,
  1092. ARMV7_PERFCTR_DTLB_REFILL = 0x05,
  1093. ARMV7_PERFCTR_DREAD = 0x06,
  1094. ARMV7_PERFCTR_DWRITE = 0x07,
  1095. ARMV7_PERFCTR_EXC_TAKEN = 0x09,
  1096. ARMV7_PERFCTR_EXC_EXECUTED = 0x0A,
  1097. ARMV7_PERFCTR_CID_WRITE = 0x0B,
  1098. /* ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
  1099. * It counts:
  1100. * - all branch instructions,
  1101. * - instructions that explicitly write the PC,
  1102. * - exception generating instructions.
  1103. */
  1104. ARMV7_PERFCTR_PC_WRITE = 0x0C,
  1105. ARMV7_PERFCTR_PC_IMM_BRANCH = 0x0D,
  1106. ARMV7_PERFCTR_UNALIGNED_ACCESS = 0x0F,
  1107. ARMV7_PERFCTR_PC_BRANCH_MIS_PRED = 0x10,
  1108. ARMV7_PERFCTR_CLOCK_CYCLES = 0x11,
  1109. ARMV7_PERFCTR_PC_BRANCH_MIS_USED = 0x12,
  1110. ARMV7_PERFCTR_CPU_CYCLES = 0xFF
  1111. };
  1112. /* ARMv7 Cortex-A8 specific event types */
  1113. enum armv7_a8_perf_types {
  1114. ARMV7_PERFCTR_INSTR_EXECUTED = 0x08,
  1115. ARMV7_PERFCTR_PC_PROC_RETURN = 0x0E,
  1116. ARMV7_PERFCTR_WRITE_BUFFER_FULL = 0x40,
  1117. ARMV7_PERFCTR_L2_STORE_MERGED = 0x41,
  1118. ARMV7_PERFCTR_L2_STORE_BUFF = 0x42,
  1119. ARMV7_PERFCTR_L2_ACCESS = 0x43,
  1120. ARMV7_PERFCTR_L2_CACH_MISS = 0x44,
  1121. ARMV7_PERFCTR_AXI_READ_CYCLES = 0x45,
  1122. ARMV7_PERFCTR_AXI_WRITE_CYCLES = 0x46,
  1123. ARMV7_PERFCTR_MEMORY_REPLAY = 0x47,
  1124. ARMV7_PERFCTR_UNALIGNED_ACCESS_REPLAY = 0x48,
  1125. ARMV7_PERFCTR_L1_DATA_MISS = 0x49,
  1126. ARMV7_PERFCTR_L1_INST_MISS = 0x4A,
  1127. ARMV7_PERFCTR_L1_DATA_COLORING = 0x4B,
  1128. ARMV7_PERFCTR_L1_NEON_DATA = 0x4C,
  1129. ARMV7_PERFCTR_L1_NEON_CACH_DATA = 0x4D,
  1130. ARMV7_PERFCTR_L2_NEON = 0x4E,
  1131. ARMV7_PERFCTR_L2_NEON_HIT = 0x4F,
  1132. ARMV7_PERFCTR_L1_INST = 0x50,
  1133. ARMV7_PERFCTR_PC_RETURN_MIS_PRED = 0x51,
  1134. ARMV7_PERFCTR_PC_BRANCH_FAILED = 0x52,
  1135. ARMV7_PERFCTR_PC_BRANCH_TAKEN = 0x53,
  1136. ARMV7_PERFCTR_PC_BRANCH_EXECUTED = 0x54,
  1137. ARMV7_PERFCTR_OP_EXECUTED = 0x55,
  1138. ARMV7_PERFCTR_CYCLES_INST_STALL = 0x56,
  1139. ARMV7_PERFCTR_CYCLES_INST = 0x57,
  1140. ARMV7_PERFCTR_CYCLES_NEON_DATA_STALL = 0x58,
  1141. ARMV7_PERFCTR_CYCLES_NEON_INST_STALL = 0x59,
  1142. ARMV7_PERFCTR_NEON_CYCLES = 0x5A,
  1143. ARMV7_PERFCTR_PMU0_EVENTS = 0x70,
  1144. ARMV7_PERFCTR_PMU1_EVENTS = 0x71,
  1145. ARMV7_PERFCTR_PMU_EVENTS = 0x72,
  1146. };
  1147. /* ARMv7 Cortex-A9 specific event types */
  1148. enum armv7_a9_perf_types {
  1149. ARMV7_PERFCTR_JAVA_HW_BYTECODE_EXEC = 0x40,
  1150. ARMV7_PERFCTR_JAVA_SW_BYTECODE_EXEC = 0x41,
  1151. ARMV7_PERFCTR_JAZELLE_BRANCH_EXEC = 0x42,
  1152. ARMV7_PERFCTR_COHERENT_LINE_MISS = 0x50,
  1153. ARMV7_PERFCTR_COHERENT_LINE_HIT = 0x51,
  1154. ARMV7_PERFCTR_ICACHE_DEP_STALL_CYCLES = 0x60,
  1155. ARMV7_PERFCTR_DCACHE_DEP_STALL_CYCLES = 0x61,
  1156. ARMV7_PERFCTR_TLB_MISS_DEP_STALL_CYCLES = 0x62,
  1157. ARMV7_PERFCTR_STREX_EXECUTED_PASSED = 0x63,
  1158. ARMV7_PERFCTR_STREX_EXECUTED_FAILED = 0x64,
  1159. ARMV7_PERFCTR_DATA_EVICTION = 0x65,
  1160. ARMV7_PERFCTR_ISSUE_STAGE_NO_INST = 0x66,
  1161. ARMV7_PERFCTR_ISSUE_STAGE_EMPTY = 0x67,
  1162. ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE = 0x68,
  1163. ARMV7_PERFCTR_PREDICTABLE_FUNCT_RETURNS = 0x6E,
  1164. ARMV7_PERFCTR_MAIN_UNIT_EXECUTED_INST = 0x70,
  1165. ARMV7_PERFCTR_SECOND_UNIT_EXECUTED_INST = 0x71,
  1166. ARMV7_PERFCTR_LD_ST_UNIT_EXECUTED_INST = 0x72,
  1167. ARMV7_PERFCTR_FP_EXECUTED_INST = 0x73,
  1168. ARMV7_PERFCTR_NEON_EXECUTED_INST = 0x74,
  1169. ARMV7_PERFCTR_PLD_FULL_DEP_STALL_CYCLES = 0x80,
  1170. ARMV7_PERFCTR_DATA_WR_DEP_STALL_CYCLES = 0x81,
  1171. ARMV7_PERFCTR_ITLB_MISS_DEP_STALL_CYCLES = 0x82,
  1172. ARMV7_PERFCTR_DTLB_MISS_DEP_STALL_CYCLES = 0x83,
  1173. ARMV7_PERFCTR_MICRO_ITLB_MISS_DEP_STALL_CYCLES = 0x84,
  1174. ARMV7_PERFCTR_MICRO_DTLB_MISS_DEP_STALL_CYCLES = 0x85,
  1175. ARMV7_PERFCTR_DMB_DEP_STALL_CYCLES = 0x86,
  1176. ARMV7_PERFCTR_INTGR_CLK_ENABLED_CYCLES = 0x8A,
  1177. ARMV7_PERFCTR_DATA_ENGINE_CLK_EN_CYCLES = 0x8B,
  1178. ARMV7_PERFCTR_ISB_INST = 0x90,
  1179. ARMV7_PERFCTR_DSB_INST = 0x91,
  1180. ARMV7_PERFCTR_DMB_INST = 0x92,
  1181. ARMV7_PERFCTR_EXT_INTERRUPTS = 0x93,
  1182. ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_COMPLETED = 0xA0,
  1183. ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_SKIPPED = 0xA1,
  1184. ARMV7_PERFCTR_PLE_FIFO_FLUSH = 0xA2,
  1185. ARMV7_PERFCTR_PLE_RQST_COMPLETED = 0xA3,
  1186. ARMV7_PERFCTR_PLE_FIFO_OVERFLOW = 0xA4,
  1187. ARMV7_PERFCTR_PLE_RQST_PROG = 0xA5
  1188. };
  1189. /*
  1190. * Cortex-A8 HW events mapping
  1191. *
  1192. * The hardware events that we support. We do support cache operations but
  1193. * we have harvard caches and no way to combine instruction and data
  1194. * accesses/misses in hardware.
  1195. */
  1196. static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
  1197. [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
  1198. [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
  1199. [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED,
  1200. [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED,
  1201. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
  1202. [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  1203. [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
  1204. };
  1205. static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  1206. [PERF_COUNT_HW_CACHE_OP_MAX]
  1207. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  1208. [C(L1D)] = {
  1209. /*
  1210. * The performance counters don't differentiate between read
  1211. * and write accesses/misses so this isn't strictly correct,
  1212. * but it's the best we can do. Writes and reads get
  1213. * combined.
  1214. */
  1215. [C(OP_READ)] = {
  1216. [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS,
  1217. [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL,
  1218. },
  1219. [C(OP_WRITE)] = {
  1220. [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS,
  1221. [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL,
  1222. },
  1223. [C(OP_PREFETCH)] = {
  1224. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1225. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1226. },
  1227. },
  1228. [C(L1I)] = {
  1229. [C(OP_READ)] = {
  1230. [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_INST,
  1231. [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_INST_MISS,
  1232. },
  1233. [C(OP_WRITE)] = {
  1234. [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_INST,
  1235. [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_INST_MISS,
  1236. },
  1237. [C(OP_PREFETCH)] = {
  1238. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1239. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1240. },
  1241. },
  1242. [C(LL)] = {
  1243. [C(OP_READ)] = {
  1244. [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_ACCESS,
  1245. [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACH_MISS,
  1246. },
  1247. [C(OP_WRITE)] = {
  1248. [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_ACCESS,
  1249. [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACH_MISS,
  1250. },
  1251. [C(OP_PREFETCH)] = {
  1252. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1253. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1254. },
  1255. },
  1256. [C(DTLB)] = {
  1257. /*
  1258. * Only ITLB misses and DTLB refills are supported.
  1259. * If users want the DTLB refills misses a raw counter
  1260. * must be used.
  1261. */
  1262. [C(OP_READ)] = {
  1263. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1264. [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  1265. },
  1266. [C(OP_WRITE)] = {
  1267. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1268. [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  1269. },
  1270. [C(OP_PREFETCH)] = {
  1271. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1272. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1273. },
  1274. },
  1275. [C(ITLB)] = {
  1276. [C(OP_READ)] = {
  1277. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1278. [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS,
  1279. },
  1280. [C(OP_WRITE)] = {
  1281. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1282. [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS,
  1283. },
  1284. [C(OP_PREFETCH)] = {
  1285. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1286. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1287. },
  1288. },
  1289. [C(BPU)] = {
  1290. [C(OP_READ)] = {
  1291. [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE,
  1292. [C(RESULT_MISS)]
  1293. = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  1294. },
  1295. [C(OP_WRITE)] = {
  1296. [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE,
  1297. [C(RESULT_MISS)]
  1298. = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  1299. },
  1300. [C(OP_PREFETCH)] = {
  1301. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1302. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1303. },
  1304. },
  1305. };
  1306. /*
  1307. * Cortex-A9 HW events mapping
  1308. */
  1309. static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
  1310. [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
  1311. [PERF_COUNT_HW_INSTRUCTIONS] =
  1312. ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE,
  1313. [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_COHERENT_LINE_HIT,
  1314. [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_COHERENT_LINE_MISS,
  1315. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
  1316. [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  1317. [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
  1318. };
  1319. static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  1320. [PERF_COUNT_HW_CACHE_OP_MAX]
  1321. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  1322. [C(L1D)] = {
  1323. /*
  1324. * The performance counters don't differentiate between read
  1325. * and write accesses/misses so this isn't strictly correct,
  1326. * but it's the best we can do. Writes and reads get
  1327. * combined.
  1328. */
  1329. [C(OP_READ)] = {
  1330. [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS,
  1331. [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL,
  1332. },
  1333. [C(OP_WRITE)] = {
  1334. [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS,
  1335. [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL,
  1336. },
  1337. [C(OP_PREFETCH)] = {
  1338. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1339. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1340. },
  1341. },
  1342. [C(L1I)] = {
  1343. [C(OP_READ)] = {
  1344. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1345. [C(RESULT_MISS)] = ARMV7_PERFCTR_IFETCH_MISS,
  1346. },
  1347. [C(OP_WRITE)] = {
  1348. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1349. [C(RESULT_MISS)] = ARMV7_PERFCTR_IFETCH_MISS,
  1350. },
  1351. [C(OP_PREFETCH)] = {
  1352. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1353. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1354. },
  1355. },
  1356. [C(LL)] = {
  1357. [C(OP_READ)] = {
  1358. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1359. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1360. },
  1361. [C(OP_WRITE)] = {
  1362. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1363. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1364. },
  1365. [C(OP_PREFETCH)] = {
  1366. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1367. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1368. },
  1369. },
  1370. [C(DTLB)] = {
  1371. /*
  1372. * Only ITLB misses and DTLB refills are supported.
  1373. * If users want the DTLB refills misses a raw counter
  1374. * must be used.
  1375. */
  1376. [C(OP_READ)] = {
  1377. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1378. [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  1379. },
  1380. [C(OP_WRITE)] = {
  1381. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1382. [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  1383. },
  1384. [C(OP_PREFETCH)] = {
  1385. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1386. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1387. },
  1388. },
  1389. [C(ITLB)] = {
  1390. [C(OP_READ)] = {
  1391. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1392. [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS,
  1393. },
  1394. [C(OP_WRITE)] = {
  1395. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1396. [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS,
  1397. },
  1398. [C(OP_PREFETCH)] = {
  1399. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1400. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1401. },
  1402. },
  1403. [C(BPU)] = {
  1404. [C(OP_READ)] = {
  1405. [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE,
  1406. [C(RESULT_MISS)]
  1407. = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  1408. },
  1409. [C(OP_WRITE)] = {
  1410. [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE,
  1411. [C(RESULT_MISS)]
  1412. = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  1413. },
  1414. [C(OP_PREFETCH)] = {
  1415. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1416. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1417. },
  1418. },
  1419. };
  1420. /*
  1421. * Perf Events counters
  1422. */
  1423. enum armv7_counters {
  1424. ARMV7_CYCLE_COUNTER = 1, /* Cycle counter */
  1425. ARMV7_COUNTER0 = 2, /* First event counter */
  1426. };
  1427. /*
  1428. * The cycle counter is ARMV7_CYCLE_COUNTER.
  1429. * The first event counter is ARMV7_COUNTER0.
  1430. * The last event counter is (ARMV7_COUNTER0 + armpmu->num_events - 1).
  1431. */
  1432. #define ARMV7_COUNTER_LAST (ARMV7_COUNTER0 + armpmu->num_events - 1)
  1433. /*
  1434. * ARMv7 low level PMNC access
  1435. */
  1436. /*
  1437. * Per-CPU PMNC: config reg
  1438. */
  1439. #define ARMV7_PMNC_E (1 << 0) /* Enable all counters */
  1440. #define ARMV7_PMNC_P (1 << 1) /* Reset all counters */
  1441. #define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */
  1442. #define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */
  1443. #define ARMV7_PMNC_X (1 << 4) /* Export to ETM */
  1444. #define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/
  1445. #define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */
  1446. #define ARMV7_PMNC_N_MASK 0x1f
  1447. #define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */
  1448. /*
  1449. * Available counters
  1450. */
  1451. #define ARMV7_CNT0 0 /* First event counter */
  1452. #define ARMV7_CCNT 31 /* Cycle counter */
  1453. /* Perf Event to low level counters mapping */
  1454. #define ARMV7_EVENT_CNT_TO_CNTx (ARMV7_COUNTER0 - ARMV7_CNT0)
  1455. /*
  1456. * CNTENS: counters enable reg
  1457. */
  1458. #define ARMV7_CNTENS_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
  1459. #define ARMV7_CNTENS_C (1 << ARMV7_CCNT)
  1460. /*
  1461. * CNTENC: counters disable reg
  1462. */
  1463. #define ARMV7_CNTENC_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
  1464. #define ARMV7_CNTENC_C (1 << ARMV7_CCNT)
  1465. /*
  1466. * INTENS: counters overflow interrupt enable reg
  1467. */
  1468. #define ARMV7_INTENS_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
  1469. #define ARMV7_INTENS_C (1 << ARMV7_CCNT)
  1470. /*
  1471. * INTENC: counters overflow interrupt disable reg
  1472. */
  1473. #define ARMV7_INTENC_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
  1474. #define ARMV7_INTENC_C (1 << ARMV7_CCNT)
  1475. /*
  1476. * EVTSEL: Event selection reg
  1477. */
  1478. #define ARMV7_EVTSEL_MASK 0xff /* Mask for writable bits */
  1479. /*
  1480. * SELECT: Counter selection reg
  1481. */
  1482. #define ARMV7_SELECT_MASK 0x1f /* Mask for writable bits */
  1483. /*
  1484. * FLAG: counters overflow flag status reg
  1485. */
  1486. #define ARMV7_FLAG_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
  1487. #define ARMV7_FLAG_C (1 << ARMV7_CCNT)
  1488. #define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */
  1489. #define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK
  1490. static inline unsigned long armv7_pmnc_read(void)
  1491. {
  1492. u32 val;
  1493. asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
  1494. return val;
  1495. }
  1496. static inline void armv7_pmnc_write(unsigned long val)
  1497. {
  1498. val &= ARMV7_PMNC_MASK;
  1499. asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
  1500. }
  1501. static inline int armv7_pmnc_has_overflowed(unsigned long pmnc)
  1502. {
  1503. return pmnc & ARMV7_OVERFLOWED_MASK;
  1504. }
  1505. static inline int armv7_pmnc_counter_has_overflowed(unsigned long pmnc,
  1506. enum armv7_counters counter)
  1507. {
  1508. int ret;
  1509. if (counter == ARMV7_CYCLE_COUNTER)
  1510. ret = pmnc & ARMV7_FLAG_C;
  1511. else if ((counter >= ARMV7_COUNTER0) && (counter <= ARMV7_COUNTER_LAST))
  1512. ret = pmnc & ARMV7_FLAG_P(counter);
  1513. else
  1514. pr_err("CPU%u checking wrong counter %d overflow status\n",
  1515. smp_processor_id(), counter);
  1516. return ret;
  1517. }
  1518. static inline int armv7_pmnc_select_counter(unsigned int idx)
  1519. {
  1520. u32 val;
  1521. if ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST)) {
  1522. pr_err("CPU%u selecting wrong PMNC counter"
  1523. " %d\n", smp_processor_id(), idx);
  1524. return -1;
  1525. }
  1526. val = (idx - ARMV7_EVENT_CNT_TO_CNTx) & ARMV7_SELECT_MASK;
  1527. asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val));
  1528. return idx;
  1529. }
  1530. static inline u32 armv7pmu_read_counter(int idx)
  1531. {
  1532. unsigned long value = 0;
  1533. if (idx == ARMV7_CYCLE_COUNTER)
  1534. asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
  1535. else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) {
  1536. if (armv7_pmnc_select_counter(idx) == idx)
  1537. asm volatile("mrc p15, 0, %0, c9, c13, 2"
  1538. : "=r" (value));
  1539. } else
  1540. pr_err("CPU%u reading wrong counter %d\n",
  1541. smp_processor_id(), idx);
  1542. return value;
  1543. }
  1544. static inline void armv7pmu_write_counter(int idx, u32 value)
  1545. {
  1546. if (idx == ARMV7_CYCLE_COUNTER)
  1547. asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
  1548. else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) {
  1549. if (armv7_pmnc_select_counter(idx) == idx)
  1550. asm volatile("mcr p15, 0, %0, c9, c13, 2"
  1551. : : "r" (value));
  1552. } else
  1553. pr_err("CPU%u writing wrong counter %d\n",
  1554. smp_processor_id(), idx);
  1555. }
  1556. static inline void armv7_pmnc_write_evtsel(unsigned int idx, u32 val)
  1557. {
  1558. if (armv7_pmnc_select_counter(idx) == idx) {
  1559. val &= ARMV7_EVTSEL_MASK;
  1560. asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
  1561. }
  1562. }
  1563. static inline u32 armv7_pmnc_enable_counter(unsigned int idx)
  1564. {
  1565. u32 val;
  1566. if ((idx != ARMV7_CYCLE_COUNTER) &&
  1567. ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
  1568. pr_err("CPU%u enabling wrong PMNC counter"
  1569. " %d\n", smp_processor_id(), idx);
  1570. return -1;
  1571. }
  1572. if (idx == ARMV7_CYCLE_COUNTER)
  1573. val = ARMV7_CNTENS_C;
  1574. else
  1575. val = ARMV7_CNTENS_P(idx);
  1576. asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val));
  1577. return idx;
  1578. }
  1579. static inline u32 armv7_pmnc_disable_counter(unsigned int idx)
  1580. {
  1581. u32 val;
  1582. if ((idx != ARMV7_CYCLE_COUNTER) &&
  1583. ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
  1584. pr_err("CPU%u disabling wrong PMNC counter"
  1585. " %d\n", smp_processor_id(), idx);
  1586. return -1;
  1587. }
  1588. if (idx == ARMV7_CYCLE_COUNTER)
  1589. val = ARMV7_CNTENC_C;
  1590. else
  1591. val = ARMV7_CNTENC_P(idx);
  1592. asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val));
  1593. return idx;
  1594. }
  1595. static inline u32 armv7_pmnc_enable_intens(unsigned int idx)
  1596. {
  1597. u32 val;
  1598. if ((idx != ARMV7_CYCLE_COUNTER) &&
  1599. ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
  1600. pr_err("CPU%u enabling wrong PMNC counter"
  1601. " interrupt enable %d\n", smp_processor_id(), idx);
  1602. return -1;
  1603. }
  1604. if (idx == ARMV7_CYCLE_COUNTER)
  1605. val = ARMV7_INTENS_C;
  1606. else
  1607. val = ARMV7_INTENS_P(idx);
  1608. asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val));
  1609. return idx;
  1610. }
  1611. static inline u32 armv7_pmnc_disable_intens(unsigned int idx)
  1612. {
  1613. u32 val;
  1614. if ((idx != ARMV7_CYCLE_COUNTER) &&
  1615. ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
  1616. pr_err("CPU%u disabling wrong PMNC counter"
  1617. " interrupt enable %d\n", smp_processor_id(), idx);
  1618. return -1;
  1619. }
  1620. if (idx == ARMV7_CYCLE_COUNTER)
  1621. val = ARMV7_INTENC_C;
  1622. else
  1623. val = ARMV7_INTENC_P(idx);
  1624. asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val));
  1625. return idx;
  1626. }
  1627. static inline u32 armv7_pmnc_getreset_flags(void)
  1628. {
  1629. u32 val;
  1630. /* Read */
  1631. asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
  1632. /* Write to clear flags */
  1633. val &= ARMV7_FLAG_MASK;
  1634. asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
  1635. return val;
  1636. }
  1637. #ifdef DEBUG
  1638. static void armv7_pmnc_dump_regs(void)
  1639. {
  1640. u32 val;
  1641. unsigned int cnt;
  1642. printk(KERN_INFO "PMNC registers dump:\n");
  1643. asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
  1644. printk(KERN_INFO "PMNC =0x%08x\n", val);
  1645. asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
  1646. printk(KERN_INFO "CNTENS=0x%08x\n", val);
  1647. asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
  1648. printk(KERN_INFO "INTENS=0x%08x\n", val);
  1649. asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
  1650. printk(KERN_INFO "FLAGS =0x%08x\n", val);
  1651. asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
  1652. printk(KERN_INFO "SELECT=0x%08x\n", val);
  1653. asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
  1654. printk(KERN_INFO "CCNT =0x%08x\n", val);
  1655. for (cnt = ARMV7_COUNTER0; cnt < ARMV7_COUNTER_LAST; cnt++) {
  1656. armv7_pmnc_select_counter(cnt);
  1657. asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
  1658. printk(KERN_INFO "CNT[%d] count =0x%08x\n",
  1659. cnt-ARMV7_EVENT_CNT_TO_CNTx, val);
  1660. asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
  1661. printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n",
  1662. cnt-ARMV7_EVENT_CNT_TO_CNTx, val);
  1663. }
  1664. }
  1665. #endif
  1666. void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx)
  1667. {
  1668. unsigned long flags;
  1669. /*
  1670. * Enable counter and interrupt, and set the counter to count
  1671. * the event that we're interested in.
  1672. */
  1673. spin_lock_irqsave(&pmu_lock, flags);
  1674. /*
  1675. * Disable counter
  1676. */
  1677. armv7_pmnc_disable_counter(idx);
  1678. /*
  1679. * Set event (if destined for PMNx counters)
  1680. * We don't need to set the event if it's a cycle count
  1681. */
  1682. if (idx != ARMV7_CYCLE_COUNTER)
  1683. armv7_pmnc_write_evtsel(idx, hwc->config_base);
  1684. /*
  1685. * Enable interrupt for this counter
  1686. */
  1687. armv7_pmnc_enable_intens(idx);
  1688. /*
  1689. * Enable counter
  1690. */
  1691. armv7_pmnc_enable_counter(idx);
  1692. spin_unlock_irqrestore(&pmu_lock, flags);
  1693. }
  1694. static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx)
  1695. {
  1696. unsigned long flags;
  1697. /*
  1698. * Disable counter and interrupt
  1699. */
  1700. spin_lock_irqsave(&pmu_lock, flags);
  1701. /*
  1702. * Disable counter
  1703. */
  1704. armv7_pmnc_disable_counter(idx);
  1705. /*
  1706. * Disable interrupt for this counter
  1707. */
  1708. armv7_pmnc_disable_intens(idx);
  1709. spin_unlock_irqrestore(&pmu_lock, flags);
  1710. }
  1711. static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
  1712. {
  1713. unsigned long pmnc;
  1714. struct perf_sample_data data;
  1715. struct cpu_hw_events *cpuc;
  1716. struct pt_regs *regs;
  1717. int idx;
  1718. /*
  1719. * Get and reset the IRQ flags
  1720. */
  1721. pmnc = armv7_pmnc_getreset_flags();
  1722. /*
  1723. * Did an overflow occur?
  1724. */
  1725. if (!armv7_pmnc_has_overflowed(pmnc))
  1726. return IRQ_NONE;
  1727. /*
  1728. * Handle the counter(s) overflow(s)
  1729. */
  1730. regs = get_irq_regs();
  1731. perf_sample_data_init(&data, 0);
  1732. cpuc = &__get_cpu_var(cpu_hw_events);
  1733. for (idx = 0; idx <= armpmu->num_events; ++idx) {
  1734. struct perf_event *event = cpuc->events[idx];
  1735. struct hw_perf_event *hwc;
  1736. if (!test_bit(idx, cpuc->active_mask))
  1737. continue;
  1738. /*
  1739. * We have a single interrupt for all counters. Check that
  1740. * each counter has overflowed before we process it.
  1741. */
  1742. if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
  1743. continue;
  1744. hwc = &event->hw;
  1745. armpmu_event_update(event, hwc, idx);
  1746. data.period = event->hw.last_period;
  1747. if (!armpmu_event_set_period(event, hwc, idx))
  1748. continue;
  1749. if (perf_event_overflow(event, 0, &data, regs))
  1750. armpmu->disable(hwc, idx);
  1751. }
  1752. /*
  1753. * Handle the pending perf events.
  1754. *
  1755. * Note: this call *must* be run with interrupts disabled. For
  1756. * platforms that can have the PMU interrupts raised as an NMI, this
  1757. * will not work.
  1758. */
  1759. perf_event_do_pending();
  1760. return IRQ_HANDLED;
  1761. }
  1762. static void armv7pmu_start(void)
  1763. {
  1764. unsigned long flags;
  1765. spin_lock_irqsave(&pmu_lock, flags);
  1766. /* Enable all counters */
  1767. armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
  1768. spin_unlock_irqrestore(&pmu_lock, flags);
  1769. }
  1770. static void armv7pmu_stop(void)
  1771. {
  1772. unsigned long flags;
  1773. spin_lock_irqsave(&pmu_lock, flags);
  1774. /* Disable all counters */
  1775. armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
  1776. spin_unlock_irqrestore(&pmu_lock, flags);
  1777. }
  1778. static inline int armv7_a8_pmu_event_map(int config)
  1779. {
  1780. int mapping = armv7_a8_perf_map[config];
  1781. if (HW_OP_UNSUPPORTED == mapping)
  1782. mapping = -EOPNOTSUPP;
  1783. return mapping;
  1784. }
  1785. static inline int armv7_a9_pmu_event_map(int config)
  1786. {
  1787. int mapping = armv7_a9_perf_map[config];
  1788. if (HW_OP_UNSUPPORTED == mapping)
  1789. mapping = -EOPNOTSUPP;
  1790. return mapping;
  1791. }
  1792. static u64 armv7pmu_raw_event(u64 config)
  1793. {
  1794. return config & 0xff;
  1795. }
  1796. static int armv7pmu_get_event_idx(struct cpu_hw_events *cpuc,
  1797. struct hw_perf_event *event)
  1798. {
  1799. int idx;
  1800. /* Always place a cycle counter into the cycle counter. */
  1801. if (event->config_base == ARMV7_PERFCTR_CPU_CYCLES) {
  1802. if (test_and_set_bit(ARMV7_CYCLE_COUNTER, cpuc->used_mask))
  1803. return -EAGAIN;
  1804. return ARMV7_CYCLE_COUNTER;
  1805. } else {
  1806. /*
  1807. * For anything other than a cycle counter, try and use
  1808. * the events counters
  1809. */
  1810. for (idx = ARMV7_COUNTER0; idx <= armpmu->num_events; ++idx) {
  1811. if (!test_and_set_bit(idx, cpuc->used_mask))
  1812. return idx;
  1813. }
  1814. /* The counters are all in use. */
  1815. return -EAGAIN;
  1816. }
  1817. }
  1818. static struct arm_pmu armv7pmu = {
  1819. .handle_irq = armv7pmu_handle_irq,
  1820. .enable = armv7pmu_enable_event,
  1821. .disable = armv7pmu_disable_event,
  1822. .raw_event = armv7pmu_raw_event,
  1823. .read_counter = armv7pmu_read_counter,
  1824. .write_counter = armv7pmu_write_counter,
  1825. .get_event_idx = armv7pmu_get_event_idx,
  1826. .start = armv7pmu_start,
  1827. .stop = armv7pmu_stop,
  1828. .max_period = (1LLU << 32) - 1,
  1829. };
  1830. static u32 __init armv7_reset_read_pmnc(void)
  1831. {
  1832. u32 nb_cnt;
  1833. /* Initialize & Reset PMNC: C and P bits */
  1834. armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
  1835. /* Read the nb of CNTx counters supported from PMNC */
  1836. nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
  1837. /* Add the CPU cycles counter and return */
  1838. return nb_cnt + 1;
  1839. }
  1840. /*
  1841. * ARMv5 [xscale] Performance counter handling code.
  1842. *
  1843. * Based on xscale OProfile code.
  1844. *
  1845. * There are two variants of the xscale PMU that we support:
  1846. * - xscale1pmu: 2 event counters and a cycle counter
  1847. * - xscale2pmu: 4 event counters and a cycle counter
  1848. * The two variants share event definitions, but have different
  1849. * PMU structures.
  1850. */
  1851. enum xscale_perf_types {
  1852. XSCALE_PERFCTR_ICACHE_MISS = 0x00,
  1853. XSCALE_PERFCTR_ICACHE_NO_DELIVER = 0x01,
  1854. XSCALE_PERFCTR_DATA_STALL = 0x02,
  1855. XSCALE_PERFCTR_ITLB_MISS = 0x03,
  1856. XSCALE_PERFCTR_DTLB_MISS = 0x04,
  1857. XSCALE_PERFCTR_BRANCH = 0x05,
  1858. XSCALE_PERFCTR_BRANCH_MISS = 0x06,
  1859. XSCALE_PERFCTR_INSTRUCTION = 0x07,
  1860. XSCALE_PERFCTR_DCACHE_FULL_STALL = 0x08,
  1861. XSCALE_PERFCTR_DCACHE_FULL_STALL_CONTIG = 0x09,
  1862. XSCALE_PERFCTR_DCACHE_ACCESS = 0x0A,
  1863. XSCALE_PERFCTR_DCACHE_MISS = 0x0B,
  1864. XSCALE_PERFCTR_DCACHE_WRITE_BACK = 0x0C,
  1865. XSCALE_PERFCTR_PC_CHANGED = 0x0D,
  1866. XSCALE_PERFCTR_BCU_REQUEST = 0x10,
  1867. XSCALE_PERFCTR_BCU_FULL = 0x11,
  1868. XSCALE_PERFCTR_BCU_DRAIN = 0x12,
  1869. XSCALE_PERFCTR_BCU_ECC_NO_ELOG = 0x14,
  1870. XSCALE_PERFCTR_BCU_1_BIT_ERR = 0x15,
  1871. XSCALE_PERFCTR_RMW = 0x16,
  1872. /* XSCALE_PERFCTR_CCNT is not hardware defined */
  1873. XSCALE_PERFCTR_CCNT = 0xFE,
  1874. XSCALE_PERFCTR_UNUSED = 0xFF,
  1875. };
  1876. enum xscale_counters {
  1877. XSCALE_CYCLE_COUNTER = 1,
  1878. XSCALE_COUNTER0,
  1879. XSCALE_COUNTER1,
  1880. XSCALE_COUNTER2,
  1881. XSCALE_COUNTER3,
  1882. };
  1883. static const unsigned xscale_perf_map[PERF_COUNT_HW_MAX] = {
  1884. [PERF_COUNT_HW_CPU_CYCLES] = XSCALE_PERFCTR_CCNT,
  1885. [PERF_COUNT_HW_INSTRUCTIONS] = XSCALE_PERFCTR_INSTRUCTION,
  1886. [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED,
  1887. [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED,
  1888. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = XSCALE_PERFCTR_BRANCH,
  1889. [PERF_COUNT_HW_BRANCH_MISSES] = XSCALE_PERFCTR_BRANCH_MISS,
  1890. [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED,
  1891. };
  1892. static const unsigned xscale_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  1893. [PERF_COUNT_HW_CACHE_OP_MAX]
  1894. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  1895. [C(L1D)] = {
  1896. [C(OP_READ)] = {
  1897. [C(RESULT_ACCESS)] = XSCALE_PERFCTR_DCACHE_ACCESS,
  1898. [C(RESULT_MISS)] = XSCALE_PERFCTR_DCACHE_MISS,
  1899. },
  1900. [C(OP_WRITE)] = {
  1901. [C(RESULT_ACCESS)] = XSCALE_PERFCTR_DCACHE_ACCESS,
  1902. [C(RESULT_MISS)] = XSCALE_PERFCTR_DCACHE_MISS,
  1903. },
  1904. [C(OP_PREFETCH)] = {
  1905. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1906. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1907. },
  1908. },
  1909. [C(L1I)] = {
  1910. [C(OP_READ)] = {
  1911. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1912. [C(RESULT_MISS)] = XSCALE_PERFCTR_ICACHE_MISS,
  1913. },
  1914. [C(OP_WRITE)] = {
  1915. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1916. [C(RESULT_MISS)] = XSCALE_PERFCTR_ICACHE_MISS,
  1917. },
  1918. [C(OP_PREFETCH)] = {
  1919. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1920. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1921. },
  1922. },
  1923. [C(LL)] = {
  1924. [C(OP_READ)] = {
  1925. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1926. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1927. },
  1928. [C(OP_WRITE)] = {
  1929. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1930. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1931. },
  1932. [C(OP_PREFETCH)] = {
  1933. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1934. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1935. },
  1936. },
  1937. [C(DTLB)] = {
  1938. [C(OP_READ)] = {
  1939. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1940. [C(RESULT_MISS)] = XSCALE_PERFCTR_DTLB_MISS,
  1941. },
  1942. [C(OP_WRITE)] = {
  1943. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1944. [C(RESULT_MISS)] = XSCALE_PERFCTR_DTLB_MISS,
  1945. },
  1946. [C(OP_PREFETCH)] = {
  1947. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1948. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1949. },
  1950. },
  1951. [C(ITLB)] = {
  1952. [C(OP_READ)] = {
  1953. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1954. [C(RESULT_MISS)] = XSCALE_PERFCTR_ITLB_MISS,
  1955. },
  1956. [C(OP_WRITE)] = {
  1957. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1958. [C(RESULT_MISS)] = XSCALE_PERFCTR_ITLB_MISS,
  1959. },
  1960. [C(OP_PREFETCH)] = {
  1961. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1962. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1963. },
  1964. },
  1965. [C(BPU)] = {
  1966. [C(OP_READ)] = {
  1967. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1968. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1969. },
  1970. [C(OP_WRITE)] = {
  1971. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1972. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1973. },
  1974. [C(OP_PREFETCH)] = {
  1975. [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
  1976. [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
  1977. },
  1978. },
  1979. };
  1980. #define XSCALE_PMU_ENABLE 0x001
  1981. #define XSCALE_PMN_RESET 0x002
  1982. #define XSCALE_CCNT_RESET 0x004
  1983. #define XSCALE_PMU_RESET (CCNT_RESET | PMN_RESET)
  1984. #define XSCALE_PMU_CNT64 0x008
  1985. static inline int
  1986. xscalepmu_event_map(int config)
  1987. {
  1988. int mapping = xscale_perf_map[config];
  1989. if (HW_OP_UNSUPPORTED == mapping)
  1990. mapping = -EOPNOTSUPP;
  1991. return mapping;
  1992. }
  1993. static u64
  1994. xscalepmu_raw_event(u64 config)
  1995. {
  1996. return config & 0xff;
  1997. }
  1998. #define XSCALE1_OVERFLOWED_MASK 0x700
  1999. #define XSCALE1_CCOUNT_OVERFLOW 0x400
  2000. #define XSCALE1_COUNT0_OVERFLOW 0x100
  2001. #define XSCALE1_COUNT1_OVERFLOW 0x200
  2002. #define XSCALE1_CCOUNT_INT_EN 0x040
  2003. #define XSCALE1_COUNT0_INT_EN 0x010
  2004. #define XSCALE1_COUNT1_INT_EN 0x020
  2005. #define XSCALE1_COUNT0_EVT_SHFT 12
  2006. #define XSCALE1_COUNT0_EVT_MASK (0xff << XSCALE1_COUNT0_EVT_SHFT)
  2007. #define XSCALE1_COUNT1_EVT_SHFT 20
  2008. #define XSCALE1_COUNT1_EVT_MASK (0xff << XSCALE1_COUNT1_EVT_SHFT)
  2009. static inline u32
  2010. xscale1pmu_read_pmnc(void)
  2011. {
  2012. u32 val;
  2013. asm volatile("mrc p14, 0, %0, c0, c0, 0" : "=r" (val));
  2014. return val;
  2015. }
  2016. static inline void
  2017. xscale1pmu_write_pmnc(u32 val)
  2018. {
  2019. /* upper 4bits and 7, 11 are write-as-0 */
  2020. val &= 0xffff77f;
  2021. asm volatile("mcr p14, 0, %0, c0, c0, 0" : : "r" (val));
  2022. }
  2023. static inline int
  2024. xscale1_pmnc_counter_has_overflowed(unsigned long pmnc,
  2025. enum xscale_counters counter)
  2026. {
  2027. int ret = 0;
  2028. switch (counter) {
  2029. case XSCALE_CYCLE_COUNTER:
  2030. ret = pmnc & XSCALE1_CCOUNT_OVERFLOW;
  2031. break;
  2032. case XSCALE_COUNTER0:
  2033. ret = pmnc & XSCALE1_COUNT0_OVERFLOW;
  2034. break;
  2035. case XSCALE_COUNTER1:
  2036. ret = pmnc & XSCALE1_COUNT1_OVERFLOW;
  2037. break;
  2038. default:
  2039. WARN_ONCE(1, "invalid counter number (%d)\n", counter);
  2040. }
  2041. return ret;
  2042. }
  2043. static irqreturn_t
  2044. xscale1pmu_handle_irq(int irq_num, void *dev)
  2045. {
  2046. unsigned long pmnc;
  2047. struct perf_sample_data data;
  2048. struct cpu_hw_events *cpuc;
  2049. struct pt_regs *regs;
  2050. int idx;
  2051. /*
  2052. * NOTE: there's an A stepping erratum that states if an overflow
  2053. * bit already exists and another occurs, the previous
  2054. * Overflow bit gets cleared. There's no workaround.
  2055. * Fixed in B stepping or later.
  2056. */
  2057. pmnc = xscale1pmu_read_pmnc();
  2058. /*
  2059. * Write the value back to clear the overflow flags. Overflow
  2060. * flags remain in pmnc for use below. We also disable the PMU
  2061. * while we process the interrupt.
  2062. */
  2063. xscale1pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
  2064. if (!(pmnc & XSCALE1_OVERFLOWED_MASK))
  2065. return IRQ_NONE;
  2066. regs = get_irq_regs();
  2067. perf_sample_data_init(&data, 0);
  2068. cpuc = &__get_cpu_var(cpu_hw_events);
  2069. for (idx = 0; idx <= armpmu->num_events; ++idx) {
  2070. struct perf_event *event = cpuc->events[idx];
  2071. struct hw_perf_event *hwc;
  2072. if (!test_bit(idx, cpuc->active_mask))
  2073. continue;
  2074. if (!xscale1_pmnc_counter_has_overflowed(pmnc, idx))
  2075. continue;
  2076. hwc = &event->hw;
  2077. armpmu_event_update(event, hwc, idx);
  2078. data.period = event->hw.last_period;
  2079. if (!armpmu_event_set_period(event, hwc, idx))
  2080. continue;
  2081. if (perf_event_overflow(event, 0, &data, regs))
  2082. armpmu->disable(hwc, idx);
  2083. }
  2084. perf_event_do_pending();
  2085. /*
  2086. * Re-enable the PMU.
  2087. */
  2088. pmnc = xscale1pmu_read_pmnc() | XSCALE_PMU_ENABLE;
  2089. xscale1pmu_write_pmnc(pmnc);
  2090. return IRQ_HANDLED;
  2091. }
  2092. static void
  2093. xscale1pmu_enable_event(struct hw_perf_event *hwc, int idx)
  2094. {
  2095. unsigned long val, mask, evt, flags;
  2096. switch (idx) {
  2097. case XSCALE_CYCLE_COUNTER:
  2098. mask = 0;
  2099. evt = XSCALE1_CCOUNT_INT_EN;
  2100. break;
  2101. case XSCALE_COUNTER0:
  2102. mask = XSCALE1_COUNT0_EVT_MASK;
  2103. evt = (hwc->config_base << XSCALE1_COUNT0_EVT_SHFT) |
  2104. XSCALE1_COUNT0_INT_EN;
  2105. break;
  2106. case XSCALE_COUNTER1:
  2107. mask = XSCALE1_COUNT1_EVT_MASK;
  2108. evt = (hwc->config_base << XSCALE1_COUNT1_EVT_SHFT) |
  2109. XSCALE1_COUNT1_INT_EN;
  2110. break;
  2111. default:
  2112. WARN_ONCE(1, "invalid counter number (%d)\n", idx);
  2113. return;
  2114. }
  2115. spin_lock_irqsave(&pmu_lock, flags);
  2116. val = xscale1pmu_read_pmnc();
  2117. val &= ~mask;
  2118. val |= evt;
  2119. xscale1pmu_write_pmnc(val);
  2120. spin_unlock_irqrestore(&pmu_lock, flags);
  2121. }
  2122. static void
  2123. xscale1pmu_disable_event(struct hw_perf_event *hwc, int idx)
  2124. {
  2125. unsigned long val, mask, evt, flags;
  2126. switch (idx) {
  2127. case XSCALE_CYCLE_COUNTER:
  2128. mask = XSCALE1_CCOUNT_INT_EN;
  2129. evt = 0;
  2130. break;
  2131. case XSCALE_COUNTER0:
  2132. mask = XSCALE1_COUNT0_INT_EN | XSCALE1_COUNT0_EVT_MASK;
  2133. evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT0_EVT_SHFT;
  2134. break;
  2135. case XSCALE_COUNTER1:
  2136. mask = XSCALE1_COUNT1_INT_EN | XSCALE1_COUNT1_EVT_MASK;
  2137. evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT1_EVT_SHFT;
  2138. break;
  2139. default:
  2140. WARN_ONCE(1, "invalid counter number (%d)\n", idx);
  2141. return;
  2142. }
  2143. spin_lock_irqsave(&pmu_lock, flags);
  2144. val = xscale1pmu_read_pmnc();
  2145. val &= ~mask;
  2146. val |= evt;
  2147. xscale1pmu_write_pmnc(val);
  2148. spin_unlock_irqrestore(&pmu_lock, flags);
  2149. }
  2150. static int
  2151. xscale1pmu_get_event_idx(struct cpu_hw_events *cpuc,
  2152. struct hw_perf_event *event)
  2153. {
  2154. if (XSCALE_PERFCTR_CCNT == event->config_base) {
  2155. if (test_and_set_bit(XSCALE_CYCLE_COUNTER, cpuc->used_mask))
  2156. return -EAGAIN;
  2157. return XSCALE_CYCLE_COUNTER;
  2158. } else {
  2159. if (!test_and_set_bit(XSCALE_COUNTER1, cpuc->used_mask)) {
  2160. return XSCALE_COUNTER1;
  2161. }
  2162. if (!test_and_set_bit(XSCALE_COUNTER0, cpuc->used_mask)) {
  2163. return XSCALE_COUNTER0;
  2164. }
  2165. return -EAGAIN;
  2166. }
  2167. }
  2168. static void
  2169. xscale1pmu_start(void)
  2170. {
  2171. unsigned long flags, val;
  2172. spin_lock_irqsave(&pmu_lock, flags);
  2173. val = xscale1pmu_read_pmnc();
  2174. val |= XSCALE_PMU_ENABLE;
  2175. xscale1pmu_write_pmnc(val);
  2176. spin_unlock_irqrestore(&pmu_lock, flags);
  2177. }
  2178. static void
  2179. xscale1pmu_stop(void)
  2180. {
  2181. unsigned long flags, val;
  2182. spin_lock_irqsave(&pmu_lock, flags);
  2183. val = xscale1pmu_read_pmnc();
  2184. val &= ~XSCALE_PMU_ENABLE;
  2185. xscale1pmu_write_pmnc(val);
  2186. spin_unlock_irqrestore(&pmu_lock, flags);
  2187. }
  2188. static inline u32
  2189. xscale1pmu_read_counter(int counter)
  2190. {
  2191. u32 val = 0;
  2192. switch (counter) {
  2193. case XSCALE_CYCLE_COUNTER:
  2194. asm volatile("mrc p14, 0, %0, c1, c0, 0" : "=r" (val));
  2195. break;
  2196. case XSCALE_COUNTER0:
  2197. asm volatile("mrc p14, 0, %0, c2, c0, 0" : "=r" (val));
  2198. break;
  2199. case XSCALE_COUNTER1:
  2200. asm volatile("mrc p14, 0, %0, c3, c0, 0" : "=r" (val));
  2201. break;
  2202. }
  2203. return val;
  2204. }
  2205. static inline void
  2206. xscale1pmu_write_counter(int counter, u32 val)
  2207. {
  2208. switch (counter) {
  2209. case XSCALE_CYCLE_COUNTER:
  2210. asm volatile("mcr p14, 0, %0, c1, c0, 0" : : "r" (val));
  2211. break;
  2212. case XSCALE_COUNTER0:
  2213. asm volatile("mcr p14, 0, %0, c2, c0, 0" : : "r" (val));
  2214. break;
  2215. case XSCALE_COUNTER1:
  2216. asm volatile("mcr p14, 0, %0, c3, c0, 0" : : "r" (val));
  2217. break;
  2218. }
  2219. }
  2220. static const struct arm_pmu xscale1pmu = {
  2221. .id = ARM_PERF_PMU_ID_XSCALE1,
  2222. .handle_irq = xscale1pmu_handle_irq,
  2223. .enable = xscale1pmu_enable_event,
  2224. .disable = xscale1pmu_disable_event,
  2225. .event_map = xscalepmu_event_map,
  2226. .raw_event = xscalepmu_raw_event,
  2227. .read_counter = xscale1pmu_read_counter,
  2228. .write_counter = xscale1pmu_write_counter,
  2229. .get_event_idx = xscale1pmu_get_event_idx,
  2230. .start = xscale1pmu_start,
  2231. .stop = xscale1pmu_stop,
  2232. .num_events = 3,
  2233. .max_period = (1LLU << 32) - 1,
  2234. };
  2235. #define XSCALE2_OVERFLOWED_MASK 0x01f
  2236. #define XSCALE2_CCOUNT_OVERFLOW 0x001
  2237. #define XSCALE2_COUNT0_OVERFLOW 0x002
  2238. #define XSCALE2_COUNT1_OVERFLOW 0x004
  2239. #define XSCALE2_COUNT2_OVERFLOW 0x008
  2240. #define XSCALE2_COUNT3_OVERFLOW 0x010
  2241. #define XSCALE2_CCOUNT_INT_EN 0x001
  2242. #define XSCALE2_COUNT0_INT_EN 0x002
  2243. #define XSCALE2_COUNT1_INT_EN 0x004
  2244. #define XSCALE2_COUNT2_INT_EN 0x008
  2245. #define XSCALE2_COUNT3_INT_EN 0x010
  2246. #define XSCALE2_COUNT0_EVT_SHFT 0
  2247. #define XSCALE2_COUNT0_EVT_MASK (0xff << XSCALE2_COUNT0_EVT_SHFT)
  2248. #define XSCALE2_COUNT1_EVT_SHFT 8
  2249. #define XSCALE2_COUNT1_EVT_MASK (0xff << XSCALE2_COUNT1_EVT_SHFT)
  2250. #define XSCALE2_COUNT2_EVT_SHFT 16
  2251. #define XSCALE2_COUNT2_EVT_MASK (0xff << XSCALE2_COUNT2_EVT_SHFT)
  2252. #define XSCALE2_COUNT3_EVT_SHFT 24
  2253. #define XSCALE2_COUNT3_EVT_MASK (0xff << XSCALE2_COUNT3_EVT_SHFT)
  2254. static inline u32
  2255. xscale2pmu_read_pmnc(void)
  2256. {
  2257. u32 val;
  2258. asm volatile("mrc p14, 0, %0, c0, c1, 0" : "=r" (val));
  2259. /* bits 1-2 and 4-23 are read-unpredictable */
  2260. return val & 0xff000009;
  2261. }
  2262. static inline void
  2263. xscale2pmu_write_pmnc(u32 val)
  2264. {
  2265. /* bits 4-23 are write-as-0, 24-31 are write ignored */
  2266. val &= 0xf;
  2267. asm volatile("mcr p14, 0, %0, c0, c1, 0" : : "r" (val));
  2268. }
  2269. static inline u32
  2270. xscale2pmu_read_overflow_flags(void)
  2271. {
  2272. u32 val;
  2273. asm volatile("mrc p14, 0, %0, c5, c1, 0" : "=r" (val));
  2274. return val;
  2275. }
  2276. static inline void
  2277. xscale2pmu_write_overflow_flags(u32 val)
  2278. {
  2279. asm volatile("mcr p14, 0, %0, c5, c1, 0" : : "r" (val));
  2280. }
  2281. static inline u32
  2282. xscale2pmu_read_event_select(void)
  2283. {
  2284. u32 val;
  2285. asm volatile("mrc p14, 0, %0, c8, c1, 0" : "=r" (val));
  2286. return val;
  2287. }
  2288. static inline void
  2289. xscale2pmu_write_event_select(u32 val)
  2290. {
  2291. asm volatile("mcr p14, 0, %0, c8, c1, 0" : : "r"(val));
  2292. }
  2293. static inline u32
  2294. xscale2pmu_read_int_enable(void)
  2295. {
  2296. u32 val;
  2297. asm volatile("mrc p14, 0, %0, c4, c1, 0" : "=r" (val));
  2298. return val;
  2299. }
  2300. static void
  2301. xscale2pmu_write_int_enable(u32 val)
  2302. {
  2303. asm volatile("mcr p14, 0, %0, c4, c1, 0" : : "r" (val));
  2304. }
  2305. static inline int
  2306. xscale2_pmnc_counter_has_overflowed(unsigned long of_flags,
  2307. enum xscale_counters counter)
  2308. {
  2309. int ret = 0;
  2310. switch (counter) {
  2311. case XSCALE_CYCLE_COUNTER:
  2312. ret = of_flags & XSCALE2_CCOUNT_OVERFLOW;
  2313. break;
  2314. case XSCALE_COUNTER0:
  2315. ret = of_flags & XSCALE2_COUNT0_OVERFLOW;
  2316. break;
  2317. case XSCALE_COUNTER1:
  2318. ret = of_flags & XSCALE2_COUNT1_OVERFLOW;
  2319. break;
  2320. case XSCALE_COUNTER2:
  2321. ret = of_flags & XSCALE2_COUNT2_OVERFLOW;
  2322. break;
  2323. case XSCALE_COUNTER3:
  2324. ret = of_flags & XSCALE2_COUNT3_OVERFLOW;
  2325. break;
  2326. default:
  2327. WARN_ONCE(1, "invalid counter number (%d)\n", counter);
  2328. }
  2329. return ret;
  2330. }
  2331. static irqreturn_t
  2332. xscale2pmu_handle_irq(int irq_num, void *dev)
  2333. {
  2334. unsigned long pmnc, of_flags;
  2335. struct perf_sample_data data;
  2336. struct cpu_hw_events *cpuc;
  2337. struct pt_regs *regs;
  2338. int idx;
  2339. /* Disable the PMU. */
  2340. pmnc = xscale2pmu_read_pmnc();
  2341. xscale2pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
  2342. /* Check the overflow flag register. */
  2343. of_flags = xscale2pmu_read_overflow_flags();
  2344. if (!(of_flags & XSCALE2_OVERFLOWED_MASK))
  2345. return IRQ_NONE;
  2346. /* Clear the overflow bits. */
  2347. xscale2pmu_write_overflow_flags(of_flags);
  2348. regs = get_irq_regs();
  2349. perf_sample_data_init(&data, 0);
  2350. cpuc = &__get_cpu_var(cpu_hw_events);
  2351. for (idx = 0; idx <= armpmu->num_events; ++idx) {
  2352. struct perf_event *event = cpuc->events[idx];
  2353. struct hw_perf_event *hwc;
  2354. if (!test_bit(idx, cpuc->active_mask))
  2355. continue;
  2356. if (!xscale2_pmnc_counter_has_overflowed(pmnc, idx))
  2357. continue;
  2358. hwc = &event->hw;
  2359. armpmu_event_update(event, hwc, idx);
  2360. data.period = event->hw.last_period;
  2361. if (!armpmu_event_set_period(event, hwc, idx))
  2362. continue;
  2363. if (perf_event_overflow(event, 0, &data, regs))
  2364. armpmu->disable(hwc, idx);
  2365. }
  2366. perf_event_do_pending();
  2367. /*
  2368. * Re-enable the PMU.
  2369. */
  2370. pmnc = xscale2pmu_read_pmnc() | XSCALE_PMU_ENABLE;
  2371. xscale2pmu_write_pmnc(pmnc);
  2372. return IRQ_HANDLED;
  2373. }
  2374. static void
  2375. xscale2pmu_enable_event(struct hw_perf_event *hwc, int idx)
  2376. {
  2377. unsigned long flags, ien, evtsel;
  2378. ien = xscale2pmu_read_int_enable();
  2379. evtsel = xscale2pmu_read_event_select();
  2380. switch (idx) {
  2381. case XSCALE_CYCLE_COUNTER:
  2382. ien |= XSCALE2_CCOUNT_INT_EN;
  2383. break;
  2384. case XSCALE_COUNTER0:
  2385. ien |= XSCALE2_COUNT0_INT_EN;
  2386. evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
  2387. evtsel |= hwc->config_base << XSCALE2_COUNT0_EVT_SHFT;
  2388. break;
  2389. case XSCALE_COUNTER1:
  2390. ien |= XSCALE2_COUNT1_INT_EN;
  2391. evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
  2392. evtsel |= hwc->config_base << XSCALE2_COUNT1_EVT_SHFT;
  2393. break;
  2394. case XSCALE_COUNTER2:
  2395. ien |= XSCALE2_COUNT2_INT_EN;
  2396. evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
  2397. evtsel |= hwc->config_base << XSCALE2_COUNT2_EVT_SHFT;
  2398. break;
  2399. case XSCALE_COUNTER3:
  2400. ien |= XSCALE2_COUNT3_INT_EN;
  2401. evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
  2402. evtsel |= hwc->config_base << XSCALE2_COUNT3_EVT_SHFT;
  2403. break;
  2404. default:
  2405. WARN_ONCE(1, "invalid counter number (%d)\n", idx);
  2406. return;
  2407. }
  2408. spin_lock_irqsave(&pmu_lock, flags);
  2409. xscale2pmu_write_event_select(evtsel);
  2410. xscale2pmu_write_int_enable(ien);
  2411. spin_unlock_irqrestore(&pmu_lock, flags);
  2412. }
  2413. static void
  2414. xscale2pmu_disable_event(struct hw_perf_event *hwc, int idx)
  2415. {
  2416. unsigned long flags, ien, evtsel;
  2417. ien = xscale2pmu_read_int_enable();
  2418. evtsel = xscale2pmu_read_event_select();
  2419. switch (idx) {
  2420. case XSCALE_CYCLE_COUNTER:
  2421. ien &= ~XSCALE2_CCOUNT_INT_EN;
  2422. break;
  2423. case XSCALE_COUNTER0:
  2424. ien &= ~XSCALE2_COUNT0_INT_EN;
  2425. evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
  2426. evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT0_EVT_SHFT;
  2427. break;
  2428. case XSCALE_COUNTER1:
  2429. ien &= ~XSCALE2_COUNT1_INT_EN;
  2430. evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
  2431. evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT1_EVT_SHFT;
  2432. break;
  2433. case XSCALE_COUNTER2:
  2434. ien &= ~XSCALE2_COUNT2_INT_EN;
  2435. evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
  2436. evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT2_EVT_SHFT;
  2437. break;
  2438. case XSCALE_COUNTER3:
  2439. ien &= ~XSCALE2_COUNT3_INT_EN;
  2440. evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
  2441. evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT3_EVT_SHFT;
  2442. break;
  2443. default:
  2444. WARN_ONCE(1, "invalid counter number (%d)\n", idx);
  2445. return;
  2446. }
  2447. spin_lock_irqsave(&pmu_lock, flags);
  2448. xscale2pmu_write_event_select(evtsel);
  2449. xscale2pmu_write_int_enable(ien);
  2450. spin_unlock_irqrestore(&pmu_lock, flags);
  2451. }
  2452. static int
  2453. xscale2pmu_get_event_idx(struct cpu_hw_events *cpuc,
  2454. struct hw_perf_event *event)
  2455. {
  2456. int idx = xscale1pmu_get_event_idx(cpuc, event);
  2457. if (idx >= 0)
  2458. goto out;
  2459. if (!test_and_set_bit(XSCALE_COUNTER3, cpuc->used_mask))
  2460. idx = XSCALE_COUNTER3;
  2461. else if (!test_and_set_bit(XSCALE_COUNTER2, cpuc->used_mask))
  2462. idx = XSCALE_COUNTER2;
  2463. out:
  2464. return idx;
  2465. }
  2466. static void
  2467. xscale2pmu_start(void)
  2468. {
  2469. unsigned long flags, val;
  2470. spin_lock_irqsave(&pmu_lock, flags);
  2471. val = xscale2pmu_read_pmnc() & ~XSCALE_PMU_CNT64;
  2472. val |= XSCALE_PMU_ENABLE;
  2473. xscale2pmu_write_pmnc(val);
  2474. spin_unlock_irqrestore(&pmu_lock, flags);
  2475. }
  2476. static void
  2477. xscale2pmu_stop(void)
  2478. {
  2479. unsigned long flags, val;
  2480. spin_lock_irqsave(&pmu_lock, flags);
  2481. val = xscale2pmu_read_pmnc();
  2482. val &= ~XSCALE_PMU_ENABLE;
  2483. xscale2pmu_write_pmnc(val);
  2484. spin_unlock_irqrestore(&pmu_lock, flags);
  2485. }
  2486. static inline u32
  2487. xscale2pmu_read_counter(int counter)
  2488. {
  2489. u32 val = 0;
  2490. switch (counter) {
  2491. case XSCALE_CYCLE_COUNTER:
  2492. asm volatile("mrc p14, 0, %0, c1, c1, 0" : "=r" (val));
  2493. break;
  2494. case XSCALE_COUNTER0:
  2495. asm volatile("mrc p14, 0, %0, c0, c2, 0" : "=r" (val));
  2496. break;
  2497. case XSCALE_COUNTER1:
  2498. asm volatile("mrc p14, 0, %0, c1, c2, 0" : "=r" (val));
  2499. break;
  2500. case XSCALE_COUNTER2:
  2501. asm volatile("mrc p14, 0, %0, c2, c2, 0" : "=r" (val));
  2502. break;
  2503. case XSCALE_COUNTER3:
  2504. asm volatile("mrc p14, 0, %0, c3, c2, 0" : "=r" (val));
  2505. break;
  2506. }
  2507. return val;
  2508. }
  2509. static inline void
  2510. xscale2pmu_write_counter(int counter, u32 val)
  2511. {
  2512. switch (counter) {
  2513. case XSCALE_CYCLE_COUNTER:
  2514. asm volatile("mcr p14, 0, %0, c1, c1, 0" : : "r" (val));
  2515. break;
  2516. case XSCALE_COUNTER0:
  2517. asm volatile("mcr p14, 0, %0, c0, c2, 0" : : "r" (val));
  2518. break;
  2519. case XSCALE_COUNTER1:
  2520. asm volatile("mcr p14, 0, %0, c1, c2, 0" : : "r" (val));
  2521. break;
  2522. case XSCALE_COUNTER2:
  2523. asm volatile("mcr p14, 0, %0, c2, c2, 0" : : "r" (val));
  2524. break;
  2525. case XSCALE_COUNTER3:
  2526. asm volatile("mcr p14, 0, %0, c3, c2, 0" : : "r" (val));
  2527. break;
  2528. }
  2529. }
  2530. static const struct arm_pmu xscale2pmu = {
  2531. .id = ARM_PERF_PMU_ID_XSCALE2,
  2532. .handle_irq = xscale2pmu_handle_irq,
  2533. .enable = xscale2pmu_enable_event,
  2534. .disable = xscale2pmu_disable_event,
  2535. .event_map = xscalepmu_event_map,
  2536. .raw_event = xscalepmu_raw_event,
  2537. .read_counter = xscale2pmu_read_counter,
  2538. .write_counter = xscale2pmu_write_counter,
  2539. .get_event_idx = xscale2pmu_get_event_idx,
  2540. .start = xscale2pmu_start,
  2541. .stop = xscale2pmu_stop,
  2542. .num_events = 5,
  2543. .max_period = (1LLU << 32) - 1,
  2544. };
  2545. static int __init
  2546. init_hw_perf_events(void)
  2547. {
  2548. unsigned long cpuid = read_cpuid_id();
  2549. unsigned long implementor = (cpuid & 0xFF000000) >> 24;
  2550. unsigned long part_number = (cpuid & 0xFFF0);
  2551. /* ARM Ltd CPUs. */
  2552. if (0x41 == implementor) {
  2553. switch (part_number) {
  2554. case 0xB360: /* ARM1136 */
  2555. case 0xB560: /* ARM1156 */
  2556. case 0xB760: /* ARM1176 */
  2557. armpmu = &armv6pmu;
  2558. memcpy(armpmu_perf_cache_map, armv6_perf_cache_map,
  2559. sizeof(armv6_perf_cache_map));
  2560. perf_max_events = armv6pmu.num_events;
  2561. break;
  2562. case 0xB020: /* ARM11mpcore */
  2563. armpmu = &armv6mpcore_pmu;
  2564. memcpy(armpmu_perf_cache_map,
  2565. armv6mpcore_perf_cache_map,
  2566. sizeof(armv6mpcore_perf_cache_map));
  2567. perf_max_events = armv6mpcore_pmu.num_events;
  2568. break;
  2569. case 0xC080: /* Cortex-A8 */
  2570. armv7pmu.id = ARM_PERF_PMU_ID_CA8;
  2571. memcpy(armpmu_perf_cache_map, armv7_a8_perf_cache_map,
  2572. sizeof(armv7_a8_perf_cache_map));
  2573. armv7pmu.event_map = armv7_a8_pmu_event_map;
  2574. armpmu = &armv7pmu;
  2575. /* Reset PMNC and read the nb of CNTx counters
  2576. supported */
  2577. armv7pmu.num_events = armv7_reset_read_pmnc();
  2578. perf_max_events = armv7pmu.num_events;
  2579. break;
  2580. case 0xC090: /* Cortex-A9 */
  2581. armv7pmu.id = ARM_PERF_PMU_ID_CA9;
  2582. memcpy(armpmu_perf_cache_map, armv7_a9_perf_cache_map,
  2583. sizeof(armv7_a9_perf_cache_map));
  2584. armv7pmu.event_map = armv7_a9_pmu_event_map;
  2585. armpmu = &armv7pmu;
  2586. /* Reset PMNC and read the nb of CNTx counters
  2587. supported */
  2588. armv7pmu.num_events = armv7_reset_read_pmnc();
  2589. perf_max_events = armv7pmu.num_events;
  2590. break;
  2591. }
  2592. /* Intel CPUs [xscale]. */
  2593. } else if (0x69 == implementor) {
  2594. part_number = (cpuid >> 13) & 0x7;
  2595. switch (part_number) {
  2596. case 1:
  2597. armpmu = &xscale1pmu;
  2598. memcpy(armpmu_perf_cache_map, xscale_perf_cache_map,
  2599. sizeof(xscale_perf_cache_map));
  2600. perf_max_events = xscale1pmu.num_events;
  2601. break;
  2602. case 2:
  2603. armpmu = &xscale2pmu;
  2604. memcpy(armpmu_perf_cache_map, xscale_perf_cache_map,
  2605. sizeof(xscale_perf_cache_map));
  2606. perf_max_events = xscale2pmu.num_events;
  2607. break;
  2608. }
  2609. }
  2610. if (armpmu) {
  2611. pr_info("enabled with %s PMU driver, %d counters available\n",
  2612. arm_pmu_names[armpmu->id], armpmu->num_events);
  2613. } else {
  2614. pr_info("no hardware support available\n");
  2615. perf_max_events = -1;
  2616. }
  2617. return 0;
  2618. }
  2619. arch_initcall(init_hw_perf_events);
  2620. /*
  2621. * Callchain handling code.
  2622. */
  2623. static inline void
  2624. callchain_store(struct perf_callchain_entry *entry,
  2625. u64 ip)
  2626. {
  2627. if (entry->nr < PERF_MAX_STACK_DEPTH)
  2628. entry->ip[entry->nr++] = ip;
  2629. }
  2630. /*
  2631. * The registers we're interested in are at the end of the variable
  2632. * length saved register structure. The fp points at the end of this
  2633. * structure so the address of this struct is:
  2634. * (struct frame_tail *)(xxx->fp)-1
  2635. *
  2636. * This code has been adapted from the ARM OProfile support.
  2637. */
  2638. struct frame_tail {
  2639. struct frame_tail *fp;
  2640. unsigned long sp;
  2641. unsigned long lr;
  2642. } __attribute__((packed));
  2643. /*
  2644. * Get the return address for a single stackframe and return a pointer to the
  2645. * next frame tail.
  2646. */
  2647. static struct frame_tail *
  2648. user_backtrace(struct frame_tail *tail,
  2649. struct perf_callchain_entry *entry)
  2650. {
  2651. struct frame_tail buftail;
  2652. /* Also check accessibility of one struct frame_tail beyond */
  2653. if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
  2654. return NULL;
  2655. if (__copy_from_user_inatomic(&buftail, tail, sizeof(buftail)))
  2656. return NULL;
  2657. callchain_store(entry, buftail.lr);
  2658. /*
  2659. * Frame pointers should strictly progress back up the stack
  2660. * (towards higher addresses).
  2661. */
  2662. if (tail >= buftail.fp)
  2663. return NULL;
  2664. return buftail.fp - 1;
  2665. }
  2666. static void
  2667. perf_callchain_user(struct pt_regs *regs,
  2668. struct perf_callchain_entry *entry)
  2669. {
  2670. struct frame_tail *tail;
  2671. callchain_store(entry, PERF_CONTEXT_USER);
  2672. if (!user_mode(regs))
  2673. regs = task_pt_regs(current);
  2674. tail = (struct frame_tail *)regs->ARM_fp - 1;
  2675. while (tail && !((unsigned long)tail & 0x3))
  2676. tail = user_backtrace(tail, entry);
  2677. }
  2678. /*
  2679. * Gets called by walk_stackframe() for every stackframe. This will be called
  2680. * whist unwinding the stackframe and is like a subroutine return so we use
  2681. * the PC.
  2682. */
  2683. static int
  2684. callchain_trace(struct stackframe *fr,
  2685. void *data)
  2686. {
  2687. struct perf_callchain_entry *entry = data;
  2688. callchain_store(entry, fr->pc);
  2689. return 0;
  2690. }
  2691. static void
  2692. perf_callchain_kernel(struct pt_regs *regs,
  2693. struct perf_callchain_entry *entry)
  2694. {
  2695. struct stackframe fr;
  2696. callchain_store(entry, PERF_CONTEXT_KERNEL);
  2697. fr.fp = regs->ARM_fp;
  2698. fr.sp = regs->ARM_sp;
  2699. fr.lr = regs->ARM_lr;
  2700. fr.pc = regs->ARM_pc;
  2701. walk_stackframe(&fr, callchain_trace, entry);
  2702. }
  2703. static void
  2704. perf_do_callchain(struct pt_regs *regs,
  2705. struct perf_callchain_entry *entry)
  2706. {
  2707. int is_user;
  2708. if (!regs)
  2709. return;
  2710. is_user = user_mode(regs);
  2711. if (!current || !current->pid)
  2712. return;
  2713. if (is_user && current->state != TASK_RUNNING)
  2714. return;
  2715. if (!is_user)
  2716. perf_callchain_kernel(regs, entry);
  2717. if (current->mm)
  2718. perf_callchain_user(regs, entry);
  2719. }
  2720. static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
  2721. struct perf_callchain_entry *
  2722. perf_callchain(struct pt_regs *regs)
  2723. {
  2724. struct perf_callchain_entry *entry = &__get_cpu_var(pmc_irq_entry);
  2725. entry->nr = 0;
  2726. perf_do_callchain(regs, entry);
  2727. return entry;
  2728. }