ring_buffer.c 79 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238
  1. /*
  2. * Generic ring buffer
  3. *
  4. * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
  5. */
  6. #include <linux/ring_buffer.h>
  7. #include <linux/trace_clock.h>
  8. #include <linux/ftrace_irq.h>
  9. #include <linux/spinlock.h>
  10. #include <linux/debugfs.h>
  11. #include <linux/uaccess.h>
  12. #include <linux/hardirq.h>
  13. #include <linux/kmemcheck.h>
  14. #include <linux/module.h>
  15. #include <linux/percpu.h>
  16. #include <linux/mutex.h>
  17. #include <linux/init.h>
  18. #include <linux/hash.h>
  19. #include <linux/list.h>
  20. #include <linux/cpu.h>
  21. #include <linux/fs.h>
  22. #include "trace.h"
  23. /*
  24. * The ring buffer header is special. We must manually up keep it.
  25. */
  26. int ring_buffer_print_entry_header(struct trace_seq *s)
  27. {
  28. int ret;
  29. ret = trace_seq_printf(s, "# compressed entry header\n");
  30. ret = trace_seq_printf(s, "\ttype_len : 5 bits\n");
  31. ret = trace_seq_printf(s, "\ttime_delta : 27 bits\n");
  32. ret = trace_seq_printf(s, "\tarray : 32 bits\n");
  33. ret = trace_seq_printf(s, "\n");
  34. ret = trace_seq_printf(s, "\tpadding : type == %d\n",
  35. RINGBUF_TYPE_PADDING);
  36. ret = trace_seq_printf(s, "\ttime_extend : type == %d\n",
  37. RINGBUF_TYPE_TIME_EXTEND);
  38. ret = trace_seq_printf(s, "\tdata max type_len == %d\n",
  39. RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
  40. return ret;
  41. }
  42. /*
  43. * The ring buffer is made up of a list of pages. A separate list of pages is
  44. * allocated for each CPU. A writer may only write to a buffer that is
  45. * associated with the CPU it is currently executing on. A reader may read
  46. * from any per cpu buffer.
  47. *
  48. * The reader is special. For each per cpu buffer, the reader has its own
  49. * reader page. When a reader has read the entire reader page, this reader
  50. * page is swapped with another page in the ring buffer.
  51. *
  52. * Now, as long as the writer is off the reader page, the reader can do what
  53. * ever it wants with that page. The writer will never write to that page
  54. * again (as long as it is out of the ring buffer).
  55. *
  56. * Here's some silly ASCII art.
  57. *
  58. * +------+
  59. * |reader| RING BUFFER
  60. * |page |
  61. * +------+ +---+ +---+ +---+
  62. * | |-->| |-->| |
  63. * +---+ +---+ +---+
  64. * ^ |
  65. * | |
  66. * +---------------+
  67. *
  68. *
  69. * +------+
  70. * |reader| RING BUFFER
  71. * |page |------------------v
  72. * +------+ +---+ +---+ +---+
  73. * | |-->| |-->| |
  74. * +---+ +---+ +---+
  75. * ^ |
  76. * | |
  77. * +---------------+
  78. *
  79. *
  80. * +------+
  81. * |reader| RING BUFFER
  82. * |page |------------------v
  83. * +------+ +---+ +---+ +---+
  84. * ^ | |-->| |-->| |
  85. * | +---+ +---+ +---+
  86. * | |
  87. * | |
  88. * +------------------------------+
  89. *
  90. *
  91. * +------+
  92. * |buffer| RING BUFFER
  93. * |page |------------------v
  94. * +------+ +---+ +---+ +---+
  95. * ^ | | | |-->| |
  96. * | New +---+ +---+ +---+
  97. * | Reader------^ |
  98. * | page |
  99. * +------------------------------+
  100. *
  101. *
  102. * After we make this swap, the reader can hand this page off to the splice
  103. * code and be done with it. It can even allocate a new page if it needs to
  104. * and swap that into the ring buffer.
  105. *
  106. * We will be using cmpxchg soon to make all this lockless.
  107. *
  108. */
  109. /*
  110. * A fast way to enable or disable all ring buffers is to
  111. * call tracing_on or tracing_off. Turning off the ring buffers
  112. * prevents all ring buffers from being recorded to.
  113. * Turning this switch on, makes it OK to write to the
  114. * ring buffer, if the ring buffer is enabled itself.
  115. *
  116. * There's three layers that must be on in order to write
  117. * to the ring buffer.
  118. *
  119. * 1) This global flag must be set.
  120. * 2) The ring buffer must be enabled for recording.
  121. * 3) The per cpu buffer must be enabled for recording.
  122. *
  123. * In case of an anomaly, this global flag has a bit set that
  124. * will permantly disable all ring buffers.
  125. */
  126. /*
  127. * Global flag to disable all recording to ring buffers
  128. * This has two bits: ON, DISABLED
  129. *
  130. * ON DISABLED
  131. * ---- ----------
  132. * 0 0 : ring buffers are off
  133. * 1 0 : ring buffers are on
  134. * X 1 : ring buffers are permanently disabled
  135. */
  136. enum {
  137. RB_BUFFERS_ON_BIT = 0,
  138. RB_BUFFERS_DISABLED_BIT = 1,
  139. };
  140. enum {
  141. RB_BUFFERS_ON = 1 << RB_BUFFERS_ON_BIT,
  142. RB_BUFFERS_DISABLED = 1 << RB_BUFFERS_DISABLED_BIT,
  143. };
  144. static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON;
  145. #define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data)
  146. /**
  147. * tracing_on - enable all tracing buffers
  148. *
  149. * This function enables all tracing buffers that may have been
  150. * disabled with tracing_off.
  151. */
  152. void tracing_on(void)
  153. {
  154. set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
  155. }
  156. EXPORT_SYMBOL_GPL(tracing_on);
  157. /**
  158. * tracing_off - turn off all tracing buffers
  159. *
  160. * This function stops all tracing buffers from recording data.
  161. * It does not disable any overhead the tracers themselves may
  162. * be causing. This function simply causes all recording to
  163. * the ring buffers to fail.
  164. */
  165. void tracing_off(void)
  166. {
  167. clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
  168. }
  169. EXPORT_SYMBOL_GPL(tracing_off);
  170. /**
  171. * tracing_off_permanent - permanently disable ring buffers
  172. *
  173. * This function, once called, will disable all ring buffers
  174. * permanently.
  175. */
  176. void tracing_off_permanent(void)
  177. {
  178. set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags);
  179. }
  180. /**
  181. * tracing_is_on - show state of ring buffers enabled
  182. */
  183. int tracing_is_on(void)
  184. {
  185. return ring_buffer_flags == RB_BUFFERS_ON;
  186. }
  187. EXPORT_SYMBOL_GPL(tracing_is_on);
  188. #include "trace.h"
  189. #define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))
  190. #define RB_ALIGNMENT 4U
  191. #define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
  192. #define RB_EVNT_MIN_SIZE 8U /* two 32bit words */
  193. /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */
  194. #define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX
  195. enum {
  196. RB_LEN_TIME_EXTEND = 8,
  197. RB_LEN_TIME_STAMP = 16,
  198. };
  199. static inline int rb_null_event(struct ring_buffer_event *event)
  200. {
  201. return event->type_len == RINGBUF_TYPE_PADDING
  202. && event->time_delta == 0;
  203. }
  204. static inline int rb_discarded_event(struct ring_buffer_event *event)
  205. {
  206. return event->type_len == RINGBUF_TYPE_PADDING && event->time_delta;
  207. }
  208. static void rb_event_set_padding(struct ring_buffer_event *event)
  209. {
  210. event->type_len = RINGBUF_TYPE_PADDING;
  211. event->time_delta = 0;
  212. }
  213. static unsigned
  214. rb_event_data_length(struct ring_buffer_event *event)
  215. {
  216. unsigned length;
  217. if (event->type_len)
  218. length = event->type_len * RB_ALIGNMENT;
  219. else
  220. length = event->array[0];
  221. return length + RB_EVNT_HDR_SIZE;
  222. }
  223. /* inline for ring buffer fast paths */
  224. static unsigned
  225. rb_event_length(struct ring_buffer_event *event)
  226. {
  227. switch (event->type_len) {
  228. case RINGBUF_TYPE_PADDING:
  229. if (rb_null_event(event))
  230. /* undefined */
  231. return -1;
  232. return event->array[0] + RB_EVNT_HDR_SIZE;
  233. case RINGBUF_TYPE_TIME_EXTEND:
  234. return RB_LEN_TIME_EXTEND;
  235. case RINGBUF_TYPE_TIME_STAMP:
  236. return RB_LEN_TIME_STAMP;
  237. case RINGBUF_TYPE_DATA:
  238. return rb_event_data_length(event);
  239. default:
  240. BUG();
  241. }
  242. /* not hit */
  243. return 0;
  244. }
  245. /**
  246. * ring_buffer_event_length - return the length of the event
  247. * @event: the event to get the length of
  248. */
  249. unsigned ring_buffer_event_length(struct ring_buffer_event *event)
  250. {
  251. unsigned length = rb_event_length(event);
  252. if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
  253. return length;
  254. length -= RB_EVNT_HDR_SIZE;
  255. if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]))
  256. length -= sizeof(event->array[0]);
  257. return length;
  258. }
  259. EXPORT_SYMBOL_GPL(ring_buffer_event_length);
  260. /* inline for ring buffer fast paths */
  261. static void *
  262. rb_event_data(struct ring_buffer_event *event)
  263. {
  264. BUG_ON(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
  265. /* If length is in len field, then array[0] has the data */
  266. if (event->type_len)
  267. return (void *)&event->array[0];
  268. /* Otherwise length is in array[0] and array[1] has the data */
  269. return (void *)&event->array[1];
  270. }
  271. /**
  272. * ring_buffer_event_data - return the data of the event
  273. * @event: the event to get the data from
  274. */
  275. void *ring_buffer_event_data(struct ring_buffer_event *event)
  276. {
  277. return rb_event_data(event);
  278. }
  279. EXPORT_SYMBOL_GPL(ring_buffer_event_data);
  280. #define for_each_buffer_cpu(buffer, cpu) \
  281. for_each_cpu(cpu, buffer->cpumask)
  282. #define TS_SHIFT 27
  283. #define TS_MASK ((1ULL << TS_SHIFT) - 1)
  284. #define TS_DELTA_TEST (~TS_MASK)
  285. struct buffer_data_page {
  286. u64 time_stamp; /* page time stamp */
  287. local_t commit; /* write committed index */
  288. unsigned char data[]; /* data of buffer page */
  289. };
  290. struct buffer_page {
  291. struct list_head list; /* list of buffer pages */
  292. local_t write; /* index for next write */
  293. unsigned read; /* index for next read */
  294. local_t entries; /* entries on this page */
  295. struct buffer_data_page *page; /* Actual data page */
  296. };
  297. static void rb_init_page(struct buffer_data_page *bpage)
  298. {
  299. local_set(&bpage->commit, 0);
  300. }
  301. /**
  302. * ring_buffer_page_len - the size of data on the page.
  303. * @page: The page to read
  304. *
  305. * Returns the amount of data on the page, including buffer page header.
  306. */
  307. size_t ring_buffer_page_len(void *page)
  308. {
  309. return local_read(&((struct buffer_data_page *)page)->commit)
  310. + BUF_PAGE_HDR_SIZE;
  311. }
  312. /*
  313. * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing
  314. * this issue out.
  315. */
  316. static void free_buffer_page(struct buffer_page *bpage)
  317. {
  318. free_page((unsigned long)bpage->page);
  319. kfree(bpage);
  320. }
  321. /*
  322. * We need to fit the time_stamp delta into 27 bits.
  323. */
  324. static inline int test_time_stamp(u64 delta)
  325. {
  326. if (delta & TS_DELTA_TEST)
  327. return 1;
  328. return 0;
  329. }
  330. #define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE)
  331. /* Max payload is BUF_PAGE_SIZE - header (8bytes) */
  332. #define BUF_MAX_DATA_SIZE (BUF_PAGE_SIZE - (sizeof(u32) * 2))
  333. /* Max number of timestamps that can fit on a page */
  334. #define RB_TIMESTAMPS_PER_PAGE (BUF_PAGE_SIZE / RB_LEN_TIME_STAMP)
  335. int ring_buffer_print_page_header(struct trace_seq *s)
  336. {
  337. struct buffer_data_page field;
  338. int ret;
  339. ret = trace_seq_printf(s, "\tfield: u64 timestamp;\t"
  340. "offset:0;\tsize:%u;\n",
  341. (unsigned int)sizeof(field.time_stamp));
  342. ret = trace_seq_printf(s, "\tfield: local_t commit;\t"
  343. "offset:%u;\tsize:%u;\n",
  344. (unsigned int)offsetof(typeof(field), commit),
  345. (unsigned int)sizeof(field.commit));
  346. ret = trace_seq_printf(s, "\tfield: char data;\t"
  347. "offset:%u;\tsize:%u;\n",
  348. (unsigned int)offsetof(typeof(field), data),
  349. (unsigned int)BUF_PAGE_SIZE);
  350. return ret;
  351. }
  352. /*
  353. * head_page == tail_page && head == tail then buffer is empty.
  354. */
  355. struct ring_buffer_per_cpu {
  356. int cpu;
  357. struct ring_buffer *buffer;
  358. spinlock_t reader_lock; /* serialize readers */
  359. raw_spinlock_t lock;
  360. struct lock_class_key lock_key;
  361. struct list_head *pages;
  362. struct buffer_page *head_page; /* read from head */
  363. struct buffer_page *tail_page; /* write to tail */
  364. struct buffer_page *commit_page; /* committed pages */
  365. struct buffer_page *reader_page;
  366. unsigned long nmi_dropped;
  367. unsigned long commit_overrun;
  368. unsigned long overrun;
  369. unsigned long read;
  370. local_t entries;
  371. local_t committing;
  372. local_t commits;
  373. u64 write_stamp;
  374. u64 read_stamp;
  375. atomic_t record_disabled;
  376. };
  377. struct ring_buffer {
  378. unsigned pages;
  379. unsigned flags;
  380. int cpus;
  381. atomic_t record_disabled;
  382. cpumask_var_t cpumask;
  383. struct lock_class_key *reader_lock_key;
  384. struct mutex mutex;
  385. struct ring_buffer_per_cpu **buffers;
  386. #ifdef CONFIG_HOTPLUG_CPU
  387. struct notifier_block cpu_notify;
  388. #endif
  389. u64 (*clock)(void);
  390. };
  391. struct ring_buffer_iter {
  392. struct ring_buffer_per_cpu *cpu_buffer;
  393. unsigned long head;
  394. struct buffer_page *head_page;
  395. u64 read_stamp;
  396. };
  397. /* buffer may be either ring_buffer or ring_buffer_per_cpu */
  398. #define RB_WARN_ON(buffer, cond) \
  399. ({ \
  400. int _____ret = unlikely(cond); \
  401. if (_____ret) { \
  402. atomic_inc(&buffer->record_disabled); \
  403. WARN_ON(1); \
  404. } \
  405. _____ret; \
  406. })
  407. /* Up this if you want to test the TIME_EXTENTS and normalization */
  408. #define DEBUG_SHIFT 0
  409. static inline u64 rb_time_stamp(struct ring_buffer *buffer, int cpu)
  410. {
  411. /* shift to debug/test normalization and TIME_EXTENTS */
  412. return buffer->clock() << DEBUG_SHIFT;
  413. }
  414. u64 ring_buffer_time_stamp(struct ring_buffer *buffer, int cpu)
  415. {
  416. u64 time;
  417. preempt_disable_notrace();
  418. time = rb_time_stamp(buffer, cpu);
  419. preempt_enable_no_resched_notrace();
  420. return time;
  421. }
  422. EXPORT_SYMBOL_GPL(ring_buffer_time_stamp);
  423. void ring_buffer_normalize_time_stamp(struct ring_buffer *buffer,
  424. int cpu, u64 *ts)
  425. {
  426. /* Just stupid testing the normalize function and deltas */
  427. *ts >>= DEBUG_SHIFT;
  428. }
  429. EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
  430. /**
  431. * check_pages - integrity check of buffer pages
  432. * @cpu_buffer: CPU buffer with pages to test
  433. *
  434. * As a safety measure we check to make sure the data pages have not
  435. * been corrupted.
  436. */
  437. static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
  438. {
  439. struct list_head *head = cpu_buffer->pages;
  440. struct buffer_page *bpage, *tmp;
  441. if (RB_WARN_ON(cpu_buffer, head->next->prev != head))
  442. return -1;
  443. if (RB_WARN_ON(cpu_buffer, head->prev->next != head))
  444. return -1;
  445. list_for_each_entry_safe(bpage, tmp, head, list) {
  446. if (RB_WARN_ON(cpu_buffer,
  447. bpage->list.next->prev != &bpage->list))
  448. return -1;
  449. if (RB_WARN_ON(cpu_buffer,
  450. bpage->list.prev->next != &bpage->list))
  451. return -1;
  452. }
  453. return 0;
  454. }
  455. static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
  456. unsigned nr_pages)
  457. {
  458. struct buffer_page *bpage, *tmp;
  459. unsigned long addr;
  460. LIST_HEAD(pages);
  461. unsigned i;
  462. WARN_ON(!nr_pages);
  463. for (i = 0; i < nr_pages; i++) {
  464. bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
  465. GFP_KERNEL, cpu_to_node(cpu_buffer->cpu));
  466. if (!bpage)
  467. goto free_pages;
  468. list_add(&bpage->list, &pages);
  469. addr = __get_free_page(GFP_KERNEL);
  470. if (!addr)
  471. goto free_pages;
  472. bpage->page = (void *)addr;
  473. rb_init_page(bpage->page);
  474. }
  475. /*
  476. * The ring buffer page list is a circular list that does not
  477. * start and end with a list head. All page list items point to
  478. * other pages.
  479. */
  480. cpu_buffer->pages = pages.next;
  481. list_del(&pages);
  482. rb_check_pages(cpu_buffer);
  483. return 0;
  484. free_pages:
  485. list_for_each_entry_safe(bpage, tmp, &pages, list) {
  486. list_del_init(&bpage->list);
  487. free_buffer_page(bpage);
  488. }
  489. return -ENOMEM;
  490. }
  491. static struct ring_buffer_per_cpu *
  492. rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
  493. {
  494. struct ring_buffer_per_cpu *cpu_buffer;
  495. struct buffer_page *bpage;
  496. unsigned long addr;
  497. int ret;
  498. cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()),
  499. GFP_KERNEL, cpu_to_node(cpu));
  500. if (!cpu_buffer)
  501. return NULL;
  502. cpu_buffer->cpu = cpu;
  503. cpu_buffer->buffer = buffer;
  504. spin_lock_init(&cpu_buffer->reader_lock);
  505. lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);
  506. cpu_buffer->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
  507. bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
  508. GFP_KERNEL, cpu_to_node(cpu));
  509. if (!bpage)
  510. goto fail_free_buffer;
  511. cpu_buffer->reader_page = bpage;
  512. addr = __get_free_page(GFP_KERNEL);
  513. if (!addr)
  514. goto fail_free_reader;
  515. bpage->page = (void *)addr;
  516. rb_init_page(bpage->page);
  517. INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
  518. ret = rb_allocate_pages(cpu_buffer, buffer->pages);
  519. if (ret < 0)
  520. goto fail_free_reader;
  521. cpu_buffer->head_page
  522. = list_entry(cpu_buffer->pages, struct buffer_page, list);
  523. cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;
  524. return cpu_buffer;
  525. fail_free_reader:
  526. free_buffer_page(cpu_buffer->reader_page);
  527. fail_free_buffer:
  528. kfree(cpu_buffer);
  529. return NULL;
  530. }
  531. static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
  532. {
  533. struct list_head *head = cpu_buffer->pages;
  534. struct buffer_page *bpage, *tmp;
  535. free_buffer_page(cpu_buffer->reader_page);
  536. if (head) {
  537. list_for_each_entry_safe(bpage, tmp, head, list) {
  538. list_del_init(&bpage->list);
  539. free_buffer_page(bpage);
  540. }
  541. bpage = list_entry(head, struct buffer_page, list);
  542. free_buffer_page(bpage);
  543. }
  544. kfree(cpu_buffer);
  545. }
  546. #ifdef CONFIG_HOTPLUG_CPU
  547. static int rb_cpu_notify(struct notifier_block *self,
  548. unsigned long action, void *hcpu);
  549. #endif
  550. /**
  551. * ring_buffer_alloc - allocate a new ring_buffer
  552. * @size: the size in bytes per cpu that is needed.
  553. * @flags: attributes to set for the ring buffer.
  554. *
  555. * Currently the only flag that is available is the RB_FL_OVERWRITE
  556. * flag. This flag means that the buffer will overwrite old data
  557. * when the buffer wraps. If this flag is not set, the buffer will
  558. * drop data when the tail hits the head.
  559. */
  560. struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
  561. struct lock_class_key *key)
  562. {
  563. struct ring_buffer *buffer;
  564. int bsize;
  565. int cpu;
  566. /* keep it in its own cache line */
  567. buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),
  568. GFP_KERNEL);
  569. if (!buffer)
  570. return NULL;
  571. if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
  572. goto fail_free_buffer;
  573. buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
  574. buffer->flags = flags;
  575. buffer->clock = trace_clock_local;
  576. buffer->reader_lock_key = key;
  577. /* need at least two pages */
  578. if (buffer->pages < 2)
  579. buffer->pages = 2;
  580. /*
  581. * In case of non-hotplug cpu, if the ring-buffer is allocated
  582. * in early initcall, it will not be notified of secondary cpus.
  583. * In that off case, we need to allocate for all possible cpus.
  584. */
  585. #ifdef CONFIG_HOTPLUG_CPU
  586. get_online_cpus();
  587. cpumask_copy(buffer->cpumask, cpu_online_mask);
  588. #else
  589. cpumask_copy(buffer->cpumask, cpu_possible_mask);
  590. #endif
  591. buffer->cpus = nr_cpu_ids;
  592. bsize = sizeof(void *) * nr_cpu_ids;
  593. buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),
  594. GFP_KERNEL);
  595. if (!buffer->buffers)
  596. goto fail_free_cpumask;
  597. for_each_buffer_cpu(buffer, cpu) {
  598. buffer->buffers[cpu] =
  599. rb_allocate_cpu_buffer(buffer, cpu);
  600. if (!buffer->buffers[cpu])
  601. goto fail_free_buffers;
  602. }
  603. #ifdef CONFIG_HOTPLUG_CPU
  604. buffer->cpu_notify.notifier_call = rb_cpu_notify;
  605. buffer->cpu_notify.priority = 0;
  606. register_cpu_notifier(&buffer->cpu_notify);
  607. #endif
  608. put_online_cpus();
  609. mutex_init(&buffer->mutex);
  610. return buffer;
  611. fail_free_buffers:
  612. for_each_buffer_cpu(buffer, cpu) {
  613. if (buffer->buffers[cpu])
  614. rb_free_cpu_buffer(buffer->buffers[cpu]);
  615. }
  616. kfree(buffer->buffers);
  617. fail_free_cpumask:
  618. free_cpumask_var(buffer->cpumask);
  619. put_online_cpus();
  620. fail_free_buffer:
  621. kfree(buffer);
  622. return NULL;
  623. }
  624. EXPORT_SYMBOL_GPL(__ring_buffer_alloc);
  625. /**
  626. * ring_buffer_free - free a ring buffer.
  627. * @buffer: the buffer to free.
  628. */
  629. void
  630. ring_buffer_free(struct ring_buffer *buffer)
  631. {
  632. int cpu;
  633. get_online_cpus();
  634. #ifdef CONFIG_HOTPLUG_CPU
  635. unregister_cpu_notifier(&buffer->cpu_notify);
  636. #endif
  637. for_each_buffer_cpu(buffer, cpu)
  638. rb_free_cpu_buffer(buffer->buffers[cpu]);
  639. put_online_cpus();
  640. free_cpumask_var(buffer->cpumask);
  641. kfree(buffer);
  642. }
  643. EXPORT_SYMBOL_GPL(ring_buffer_free);
  644. void ring_buffer_set_clock(struct ring_buffer *buffer,
  645. u64 (*clock)(void))
  646. {
  647. buffer->clock = clock;
  648. }
  649. static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);
  650. static void
  651. rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages)
  652. {
  653. struct buffer_page *bpage;
  654. struct list_head *p;
  655. unsigned i;
  656. atomic_inc(&cpu_buffer->record_disabled);
  657. synchronize_sched();
  658. for (i = 0; i < nr_pages; i++) {
  659. if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages)))
  660. return;
  661. p = cpu_buffer->pages->next;
  662. bpage = list_entry(p, struct buffer_page, list);
  663. list_del_init(&bpage->list);
  664. free_buffer_page(bpage);
  665. }
  666. if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages)))
  667. return;
  668. rb_reset_cpu(cpu_buffer);
  669. rb_check_pages(cpu_buffer);
  670. atomic_dec(&cpu_buffer->record_disabled);
  671. }
  672. static void
  673. rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer,
  674. struct list_head *pages, unsigned nr_pages)
  675. {
  676. struct buffer_page *bpage;
  677. struct list_head *p;
  678. unsigned i;
  679. atomic_inc(&cpu_buffer->record_disabled);
  680. synchronize_sched();
  681. for (i = 0; i < nr_pages; i++) {
  682. if (RB_WARN_ON(cpu_buffer, list_empty(pages)))
  683. return;
  684. p = pages->next;
  685. bpage = list_entry(p, struct buffer_page, list);
  686. list_del_init(&bpage->list);
  687. list_add_tail(&bpage->list, cpu_buffer->pages);
  688. }
  689. rb_reset_cpu(cpu_buffer);
  690. rb_check_pages(cpu_buffer);
  691. atomic_dec(&cpu_buffer->record_disabled);
  692. }
  693. /**
  694. * ring_buffer_resize - resize the ring buffer
  695. * @buffer: the buffer to resize.
  696. * @size: the new size.
  697. *
  698. * The tracer is responsible for making sure that the buffer is
  699. * not being used while changing the size.
  700. * Note: We may be able to change the above requirement by using
  701. * RCU synchronizations.
  702. *
  703. * Minimum size is 2 * BUF_PAGE_SIZE.
  704. *
  705. * Returns -1 on failure.
  706. */
  707. int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
  708. {
  709. struct ring_buffer_per_cpu *cpu_buffer;
  710. unsigned nr_pages, rm_pages, new_pages;
  711. struct buffer_page *bpage, *tmp;
  712. unsigned long buffer_size;
  713. unsigned long addr;
  714. LIST_HEAD(pages);
  715. int i, cpu;
  716. /*
  717. * Always succeed at resizing a non-existent buffer:
  718. */
  719. if (!buffer)
  720. return size;
  721. size = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
  722. size *= BUF_PAGE_SIZE;
  723. buffer_size = buffer->pages * BUF_PAGE_SIZE;
  724. /* we need a minimum of two pages */
  725. if (size < BUF_PAGE_SIZE * 2)
  726. size = BUF_PAGE_SIZE * 2;
  727. if (size == buffer_size)
  728. return size;
  729. mutex_lock(&buffer->mutex);
  730. get_online_cpus();
  731. nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
  732. if (size < buffer_size) {
  733. /* easy case, just free pages */
  734. if (RB_WARN_ON(buffer, nr_pages >= buffer->pages))
  735. goto out_fail;
  736. rm_pages = buffer->pages - nr_pages;
  737. for_each_buffer_cpu(buffer, cpu) {
  738. cpu_buffer = buffer->buffers[cpu];
  739. rb_remove_pages(cpu_buffer, rm_pages);
  740. }
  741. goto out;
  742. }
  743. /*
  744. * This is a bit more difficult. We only want to add pages
  745. * when we can allocate enough for all CPUs. We do this
  746. * by allocating all the pages and storing them on a local
  747. * link list. If we succeed in our allocation, then we
  748. * add these pages to the cpu_buffers. Otherwise we just free
  749. * them all and return -ENOMEM;
  750. */
  751. if (RB_WARN_ON(buffer, nr_pages <= buffer->pages))
  752. goto out_fail;
  753. new_pages = nr_pages - buffer->pages;
  754. for_each_buffer_cpu(buffer, cpu) {
  755. for (i = 0; i < new_pages; i++) {
  756. bpage = kzalloc_node(ALIGN(sizeof(*bpage),
  757. cache_line_size()),
  758. GFP_KERNEL, cpu_to_node(cpu));
  759. if (!bpage)
  760. goto free_pages;
  761. list_add(&bpage->list, &pages);
  762. addr = __get_free_page(GFP_KERNEL);
  763. if (!addr)
  764. goto free_pages;
  765. bpage->page = (void *)addr;
  766. rb_init_page(bpage->page);
  767. }
  768. }
  769. for_each_buffer_cpu(buffer, cpu) {
  770. cpu_buffer = buffer->buffers[cpu];
  771. rb_insert_pages(cpu_buffer, &pages, new_pages);
  772. }
  773. if (RB_WARN_ON(buffer, !list_empty(&pages)))
  774. goto out_fail;
  775. out:
  776. buffer->pages = nr_pages;
  777. put_online_cpus();
  778. mutex_unlock(&buffer->mutex);
  779. return size;
  780. free_pages:
  781. list_for_each_entry_safe(bpage, tmp, &pages, list) {
  782. list_del_init(&bpage->list);
  783. free_buffer_page(bpage);
  784. }
  785. put_online_cpus();
  786. mutex_unlock(&buffer->mutex);
  787. return -ENOMEM;
  788. /*
  789. * Something went totally wrong, and we are too paranoid
  790. * to even clean up the mess.
  791. */
  792. out_fail:
  793. put_online_cpus();
  794. mutex_unlock(&buffer->mutex);
  795. return -1;
  796. }
  797. EXPORT_SYMBOL_GPL(ring_buffer_resize);
  798. static inline void *
  799. __rb_data_page_index(struct buffer_data_page *bpage, unsigned index)
  800. {
  801. return bpage->data + index;
  802. }
  803. static inline void *__rb_page_index(struct buffer_page *bpage, unsigned index)
  804. {
  805. return bpage->page->data + index;
  806. }
  807. static inline struct ring_buffer_event *
  808. rb_reader_event(struct ring_buffer_per_cpu *cpu_buffer)
  809. {
  810. return __rb_page_index(cpu_buffer->reader_page,
  811. cpu_buffer->reader_page->read);
  812. }
  813. static inline struct ring_buffer_event *
  814. rb_head_event(struct ring_buffer_per_cpu *cpu_buffer)
  815. {
  816. return __rb_page_index(cpu_buffer->head_page,
  817. cpu_buffer->head_page->read);
  818. }
  819. static inline struct ring_buffer_event *
  820. rb_iter_head_event(struct ring_buffer_iter *iter)
  821. {
  822. return __rb_page_index(iter->head_page, iter->head);
  823. }
  824. static inline unsigned rb_page_write(struct buffer_page *bpage)
  825. {
  826. return local_read(&bpage->write);
  827. }
  828. static inline unsigned rb_page_commit(struct buffer_page *bpage)
  829. {
  830. return local_read(&bpage->page->commit);
  831. }
  832. /* Size is determined by what has been commited */
  833. static inline unsigned rb_page_size(struct buffer_page *bpage)
  834. {
  835. return rb_page_commit(bpage);
  836. }
  837. static inline unsigned
  838. rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer)
  839. {
  840. return rb_page_commit(cpu_buffer->commit_page);
  841. }
  842. static inline unsigned rb_head_size(struct ring_buffer_per_cpu *cpu_buffer)
  843. {
  844. return rb_page_commit(cpu_buffer->head_page);
  845. }
  846. static inline void rb_inc_page(struct ring_buffer_per_cpu *cpu_buffer,
  847. struct buffer_page **bpage)
  848. {
  849. struct list_head *p = (*bpage)->list.next;
  850. *bpage = list_entry(p, struct buffer_page, list);
  851. }
  852. static inline unsigned
  853. rb_event_index(struct ring_buffer_event *event)
  854. {
  855. unsigned long addr = (unsigned long)event;
  856. return (addr & ~PAGE_MASK) - BUF_PAGE_HDR_SIZE;
  857. }
  858. static inline int
  859. rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
  860. struct ring_buffer_event *event)
  861. {
  862. unsigned long addr = (unsigned long)event;
  863. unsigned long index;
  864. index = rb_event_index(event);
  865. addr &= PAGE_MASK;
  866. return cpu_buffer->commit_page->page == (void *)addr &&
  867. rb_commit_index(cpu_buffer) == index;
  868. }
  869. static void
  870. rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
  871. {
  872. /*
  873. * We only race with interrupts and NMIs on this CPU.
  874. * If we own the commit event, then we can commit
  875. * all others that interrupted us, since the interruptions
  876. * are in stack format (they finish before they come
  877. * back to us). This allows us to do a simple loop to
  878. * assign the commit to the tail.
  879. */
  880. again:
  881. while (cpu_buffer->commit_page != cpu_buffer->tail_page) {
  882. cpu_buffer->commit_page->page->commit =
  883. cpu_buffer->commit_page->write;
  884. rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
  885. cpu_buffer->write_stamp =
  886. cpu_buffer->commit_page->page->time_stamp;
  887. /* add barrier to keep gcc from optimizing too much */
  888. barrier();
  889. }
  890. while (rb_commit_index(cpu_buffer) !=
  891. rb_page_write(cpu_buffer->commit_page)) {
  892. cpu_buffer->commit_page->page->commit =
  893. cpu_buffer->commit_page->write;
  894. barrier();
  895. }
  896. /* again, keep gcc from optimizing */
  897. barrier();
  898. /*
  899. * If an interrupt came in just after the first while loop
  900. * and pushed the tail page forward, we will be left with
  901. * a dangling commit that will never go forward.
  902. */
  903. if (unlikely(cpu_buffer->commit_page != cpu_buffer->tail_page))
  904. goto again;
  905. }
  906. static void rb_reset_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
  907. {
  908. cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp;
  909. cpu_buffer->reader_page->read = 0;
  910. }
  911. static void rb_inc_iter(struct ring_buffer_iter *iter)
  912. {
  913. struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
  914. /*
  915. * The iterator could be on the reader page (it starts there).
  916. * But the head could have moved, since the reader was
  917. * found. Check for this case and assign the iterator
  918. * to the head page instead of next.
  919. */
  920. if (iter->head_page == cpu_buffer->reader_page)
  921. iter->head_page = cpu_buffer->head_page;
  922. else
  923. rb_inc_page(cpu_buffer, &iter->head_page);
  924. iter->read_stamp = iter->head_page->page->time_stamp;
  925. iter->head = 0;
  926. }
  927. /**
  928. * ring_buffer_update_event - update event type and data
  929. * @event: the even to update
  930. * @type: the type of event
  931. * @length: the size of the event field in the ring buffer
  932. *
  933. * Update the type and data fields of the event. The length
  934. * is the actual size that is written to the ring buffer,
  935. * and with this, we can determine what to place into the
  936. * data field.
  937. */
  938. static void
  939. rb_update_event(struct ring_buffer_event *event,
  940. unsigned type, unsigned length)
  941. {
  942. event->type_len = type;
  943. switch (type) {
  944. case RINGBUF_TYPE_PADDING:
  945. case RINGBUF_TYPE_TIME_EXTEND:
  946. case RINGBUF_TYPE_TIME_STAMP:
  947. break;
  948. case 0:
  949. length -= RB_EVNT_HDR_SIZE;
  950. if (length > RB_MAX_SMALL_DATA)
  951. event->array[0] = length;
  952. else
  953. event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT);
  954. break;
  955. default:
  956. BUG();
  957. }
  958. }
  959. static unsigned rb_calculate_event_length(unsigned length)
  960. {
  961. struct ring_buffer_event event; /* Used only for sizeof array */
  962. /* zero length can cause confusions */
  963. if (!length)
  964. length = 1;
  965. if (length > RB_MAX_SMALL_DATA)
  966. length += sizeof(event.array[0]);
  967. length += RB_EVNT_HDR_SIZE;
  968. length = ALIGN(length, RB_ALIGNMENT);
  969. return length;
  970. }
  971. static inline void
  972. rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
  973. struct buffer_page *tail_page,
  974. unsigned long tail, unsigned long length)
  975. {
  976. struct ring_buffer_event *event;
  977. /*
  978. * Only the event that crossed the page boundary
  979. * must fill the old tail_page with padding.
  980. */
  981. if (tail >= BUF_PAGE_SIZE) {
  982. local_sub(length, &tail_page->write);
  983. return;
  984. }
  985. event = __rb_page_index(tail_page, tail);
  986. kmemcheck_annotate_bitfield(event, bitfield);
  987. /*
  988. * If this event is bigger than the minimum size, then
  989. * we need to be careful that we don't subtract the
  990. * write counter enough to allow another writer to slip
  991. * in on this page.
  992. * We put in a discarded commit instead, to make sure
  993. * that this space is not used again.
  994. *
  995. * If we are less than the minimum size, we don't need to
  996. * worry about it.
  997. */
  998. if (tail > (BUF_PAGE_SIZE - RB_EVNT_MIN_SIZE)) {
  999. /* No room for any events */
  1000. /* Mark the rest of the page with padding */
  1001. rb_event_set_padding(event);
  1002. /* Set the write back to the previous setting */
  1003. local_sub(length, &tail_page->write);
  1004. return;
  1005. }
  1006. /* Put in a discarded event */
  1007. event->array[0] = (BUF_PAGE_SIZE - tail) - RB_EVNT_HDR_SIZE;
  1008. event->type_len = RINGBUF_TYPE_PADDING;
  1009. /* time delta must be non zero */
  1010. event->time_delta = 1;
  1011. /* Account for this as an entry */
  1012. local_inc(&tail_page->entries);
  1013. local_inc(&cpu_buffer->entries);
  1014. /* Set write to end of buffer */
  1015. length = (tail + length) - BUF_PAGE_SIZE;
  1016. local_sub(length, &tail_page->write);
  1017. }
  1018. static struct ring_buffer_event *
  1019. rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
  1020. unsigned long length, unsigned long tail,
  1021. struct buffer_page *commit_page,
  1022. struct buffer_page *tail_page, u64 *ts)
  1023. {
  1024. struct buffer_page *next_page, *head_page, *reader_page;
  1025. struct ring_buffer *buffer = cpu_buffer->buffer;
  1026. bool lock_taken = false;
  1027. unsigned long flags;
  1028. next_page = tail_page;
  1029. local_irq_save(flags);
  1030. /*
  1031. * Since the write to the buffer is still not
  1032. * fully lockless, we must be careful with NMIs.
  1033. * The locks in the writers are taken when a write
  1034. * crosses to a new page. The locks protect against
  1035. * races with the readers (this will soon be fixed
  1036. * with a lockless solution).
  1037. *
  1038. * Because we can not protect against NMIs, and we
  1039. * want to keep traces reentrant, we need to manage
  1040. * what happens when we are in an NMI.
  1041. *
  1042. * NMIs can happen after we take the lock.
  1043. * If we are in an NMI, only take the lock
  1044. * if it is not already taken. Otherwise
  1045. * simply fail.
  1046. */
  1047. if (unlikely(in_nmi())) {
  1048. if (!__raw_spin_trylock(&cpu_buffer->lock)) {
  1049. cpu_buffer->nmi_dropped++;
  1050. goto out_reset;
  1051. }
  1052. } else
  1053. __raw_spin_lock(&cpu_buffer->lock);
  1054. lock_taken = true;
  1055. rb_inc_page(cpu_buffer, &next_page);
  1056. head_page = cpu_buffer->head_page;
  1057. reader_page = cpu_buffer->reader_page;
  1058. /* we grabbed the lock before incrementing */
  1059. if (RB_WARN_ON(cpu_buffer, next_page == reader_page))
  1060. goto out_reset;
  1061. /*
  1062. * If for some reason, we had an interrupt storm that made
  1063. * it all the way around the buffer, bail, and warn
  1064. * about it.
  1065. */
  1066. if (unlikely(next_page == commit_page)) {
  1067. cpu_buffer->commit_overrun++;
  1068. goto out_reset;
  1069. }
  1070. if (next_page == head_page) {
  1071. if (!(buffer->flags & RB_FL_OVERWRITE))
  1072. goto out_reset;
  1073. /* tail_page has not moved yet? */
  1074. if (tail_page == cpu_buffer->tail_page) {
  1075. /* count overflows */
  1076. cpu_buffer->overrun +=
  1077. local_read(&head_page->entries);
  1078. rb_inc_page(cpu_buffer, &head_page);
  1079. cpu_buffer->head_page = head_page;
  1080. cpu_buffer->head_page->read = 0;
  1081. }
  1082. }
  1083. /*
  1084. * If the tail page is still the same as what we think
  1085. * it is, then it is up to us to update the tail
  1086. * pointer.
  1087. */
  1088. if (tail_page == cpu_buffer->tail_page) {
  1089. local_set(&next_page->write, 0);
  1090. local_set(&next_page->entries, 0);
  1091. local_set(&next_page->page->commit, 0);
  1092. cpu_buffer->tail_page = next_page;
  1093. /* reread the time stamp */
  1094. *ts = rb_time_stamp(buffer, cpu_buffer->cpu);
  1095. cpu_buffer->tail_page->page->time_stamp = *ts;
  1096. }
  1097. rb_reset_tail(cpu_buffer, tail_page, tail, length);
  1098. __raw_spin_unlock(&cpu_buffer->lock);
  1099. local_irq_restore(flags);
  1100. /* fail and let the caller try again */
  1101. return ERR_PTR(-EAGAIN);
  1102. out_reset:
  1103. /* reset write */
  1104. rb_reset_tail(cpu_buffer, tail_page, tail, length);
  1105. if (likely(lock_taken))
  1106. __raw_spin_unlock(&cpu_buffer->lock);
  1107. local_irq_restore(flags);
  1108. return NULL;
  1109. }
  1110. static struct ring_buffer_event *
  1111. __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
  1112. unsigned type, unsigned long length, u64 *ts)
  1113. {
  1114. struct buffer_page *tail_page, *commit_page;
  1115. struct ring_buffer_event *event;
  1116. unsigned long tail, write;
  1117. commit_page = cpu_buffer->commit_page;
  1118. /* we just need to protect against interrupts */
  1119. barrier();
  1120. tail_page = cpu_buffer->tail_page;
  1121. write = local_add_return(length, &tail_page->write);
  1122. tail = write - length;
  1123. /* See if we shot pass the end of this buffer page */
  1124. if (write > BUF_PAGE_SIZE)
  1125. return rb_move_tail(cpu_buffer, length, tail,
  1126. commit_page, tail_page, ts);
  1127. /* We reserved something on the buffer */
  1128. event = __rb_page_index(tail_page, tail);
  1129. kmemcheck_annotate_bitfield(event, bitfield);
  1130. rb_update_event(event, type, length);
  1131. /* The passed in type is zero for DATA */
  1132. if (likely(!type))
  1133. local_inc(&tail_page->entries);
  1134. /*
  1135. * If this is the first commit on the page, then update
  1136. * its timestamp.
  1137. */
  1138. if (!tail)
  1139. tail_page->page->time_stamp = *ts;
  1140. return event;
  1141. }
  1142. static inline int
  1143. rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
  1144. struct ring_buffer_event *event)
  1145. {
  1146. unsigned long new_index, old_index;
  1147. struct buffer_page *bpage;
  1148. unsigned long index;
  1149. unsigned long addr;
  1150. new_index = rb_event_index(event);
  1151. old_index = new_index + rb_event_length(event);
  1152. addr = (unsigned long)event;
  1153. addr &= PAGE_MASK;
  1154. bpage = cpu_buffer->tail_page;
  1155. if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {
  1156. /*
  1157. * This is on the tail page. It is possible that
  1158. * a write could come in and move the tail page
  1159. * and write to the next page. That is fine
  1160. * because we just shorten what is on this page.
  1161. */
  1162. index = local_cmpxchg(&bpage->write, old_index, new_index);
  1163. if (index == old_index)
  1164. return 1;
  1165. }
  1166. /* could not discard */
  1167. return 0;
  1168. }
  1169. static int
  1170. rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
  1171. u64 *ts, u64 *delta)
  1172. {
  1173. struct ring_buffer_event *event;
  1174. static int once;
  1175. int ret;
  1176. if (unlikely(*delta > (1ULL << 59) && !once++)) {
  1177. printk(KERN_WARNING "Delta way too big! %llu"
  1178. " ts=%llu write stamp = %llu\n",
  1179. (unsigned long long)*delta,
  1180. (unsigned long long)*ts,
  1181. (unsigned long long)cpu_buffer->write_stamp);
  1182. WARN_ON(1);
  1183. }
  1184. /*
  1185. * The delta is too big, we to add a
  1186. * new timestamp.
  1187. */
  1188. event = __rb_reserve_next(cpu_buffer,
  1189. RINGBUF_TYPE_TIME_EXTEND,
  1190. RB_LEN_TIME_EXTEND,
  1191. ts);
  1192. if (!event)
  1193. return -EBUSY;
  1194. if (PTR_ERR(event) == -EAGAIN)
  1195. return -EAGAIN;
  1196. /* Only a commited time event can update the write stamp */
  1197. if (rb_event_is_commit(cpu_buffer, event)) {
  1198. /*
  1199. * If this is the first on the page, then it was
  1200. * updated with the page itself. Try to discard it
  1201. * and if we can't just make it zero.
  1202. */
  1203. if (rb_event_index(event)) {
  1204. event->time_delta = *delta & TS_MASK;
  1205. event->array[0] = *delta >> TS_SHIFT;
  1206. } else {
  1207. /* try to discard, since we do not need this */
  1208. if (!rb_try_to_discard(cpu_buffer, event)) {
  1209. /* nope, just zero it */
  1210. event->time_delta = 0;
  1211. event->array[0] = 0;
  1212. }
  1213. }
  1214. cpu_buffer->write_stamp = *ts;
  1215. /* let the caller know this was the commit */
  1216. ret = 1;
  1217. } else {
  1218. /* Try to discard the event */
  1219. if (!rb_try_to_discard(cpu_buffer, event)) {
  1220. /* Darn, this is just wasted space */
  1221. event->time_delta = 0;
  1222. event->array[0] = 0;
  1223. }
  1224. ret = 0;
  1225. }
  1226. *delta = 0;
  1227. return ret;
  1228. }
  1229. static void rb_start_commit(struct ring_buffer_per_cpu *cpu_buffer)
  1230. {
  1231. local_inc(&cpu_buffer->committing);
  1232. local_inc(&cpu_buffer->commits);
  1233. }
  1234. static void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer)
  1235. {
  1236. unsigned long commits;
  1237. if (RB_WARN_ON(cpu_buffer,
  1238. !local_read(&cpu_buffer->committing)))
  1239. return;
  1240. again:
  1241. commits = local_read(&cpu_buffer->commits);
  1242. /* synchronize with interrupts */
  1243. barrier();
  1244. if (local_read(&cpu_buffer->committing) == 1)
  1245. rb_set_commit_to_write(cpu_buffer);
  1246. local_dec(&cpu_buffer->committing);
  1247. /* synchronize with interrupts */
  1248. barrier();
  1249. /*
  1250. * Need to account for interrupts coming in between the
  1251. * updating of the commit page and the clearing of the
  1252. * committing counter.
  1253. */
  1254. if (unlikely(local_read(&cpu_buffer->commits) != commits) &&
  1255. !local_read(&cpu_buffer->committing)) {
  1256. local_inc(&cpu_buffer->committing);
  1257. goto again;
  1258. }
  1259. }
  1260. static struct ring_buffer_event *
  1261. rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer,
  1262. unsigned long length)
  1263. {
  1264. struct ring_buffer_event *event;
  1265. u64 ts, delta = 0;
  1266. int commit = 0;
  1267. int nr_loops = 0;
  1268. rb_start_commit(cpu_buffer);
  1269. length = rb_calculate_event_length(length);
  1270. again:
  1271. /*
  1272. * We allow for interrupts to reenter here and do a trace.
  1273. * If one does, it will cause this original code to loop
  1274. * back here. Even with heavy interrupts happening, this
  1275. * should only happen a few times in a row. If this happens
  1276. * 1000 times in a row, there must be either an interrupt
  1277. * storm or we have something buggy.
  1278. * Bail!
  1279. */
  1280. if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))
  1281. goto out_fail;
  1282. ts = rb_time_stamp(cpu_buffer->buffer, cpu_buffer->cpu);
  1283. /*
  1284. * Only the first commit can update the timestamp.
  1285. * Yes there is a race here. If an interrupt comes in
  1286. * just after the conditional and it traces too, then it
  1287. * will also check the deltas. More than one timestamp may
  1288. * also be made. But only the entry that did the actual
  1289. * commit will be something other than zero.
  1290. */
  1291. if (likely(cpu_buffer->tail_page == cpu_buffer->commit_page &&
  1292. rb_page_write(cpu_buffer->tail_page) ==
  1293. rb_commit_index(cpu_buffer))) {
  1294. u64 diff;
  1295. diff = ts - cpu_buffer->write_stamp;
  1296. /* make sure this diff is calculated here */
  1297. barrier();
  1298. /* Did the write stamp get updated already? */
  1299. if (unlikely(ts < cpu_buffer->write_stamp))
  1300. goto get_event;
  1301. delta = diff;
  1302. if (unlikely(test_time_stamp(delta))) {
  1303. commit = rb_add_time_stamp(cpu_buffer, &ts, &delta);
  1304. if (commit == -EBUSY)
  1305. goto out_fail;
  1306. if (commit == -EAGAIN)
  1307. goto again;
  1308. RB_WARN_ON(cpu_buffer, commit < 0);
  1309. }
  1310. }
  1311. get_event:
  1312. event = __rb_reserve_next(cpu_buffer, 0, length, &ts);
  1313. if (unlikely(PTR_ERR(event) == -EAGAIN))
  1314. goto again;
  1315. if (!event)
  1316. goto out_fail;
  1317. if (!rb_event_is_commit(cpu_buffer, event))
  1318. delta = 0;
  1319. event->time_delta = delta;
  1320. return event;
  1321. out_fail:
  1322. rb_end_commit(cpu_buffer);
  1323. return NULL;
  1324. }
  1325. #ifdef CONFIG_TRACING
  1326. #define TRACE_RECURSIVE_DEPTH 16
  1327. static int trace_recursive_lock(void)
  1328. {
  1329. current->trace_recursion++;
  1330. if (likely(current->trace_recursion < TRACE_RECURSIVE_DEPTH))
  1331. return 0;
  1332. /* Disable all tracing before we do anything else */
  1333. tracing_off_permanent();
  1334. printk_once(KERN_WARNING "Tracing recursion: depth[%ld]:"
  1335. "HC[%lu]:SC[%lu]:NMI[%lu]\n",
  1336. current->trace_recursion,
  1337. hardirq_count() >> HARDIRQ_SHIFT,
  1338. softirq_count() >> SOFTIRQ_SHIFT,
  1339. in_nmi());
  1340. WARN_ON_ONCE(1);
  1341. return -1;
  1342. }
  1343. static void trace_recursive_unlock(void)
  1344. {
  1345. WARN_ON_ONCE(!current->trace_recursion);
  1346. current->trace_recursion--;
  1347. }
  1348. #else
  1349. #define trace_recursive_lock() (0)
  1350. #define trace_recursive_unlock() do { } while (0)
  1351. #endif
  1352. static DEFINE_PER_CPU(int, rb_need_resched);
  1353. /**
  1354. * ring_buffer_lock_reserve - reserve a part of the buffer
  1355. * @buffer: the ring buffer to reserve from
  1356. * @length: the length of the data to reserve (excluding event header)
  1357. *
  1358. * Returns a reseverd event on the ring buffer to copy directly to.
  1359. * The user of this interface will need to get the body to write into
  1360. * and can use the ring_buffer_event_data() interface.
  1361. *
  1362. * The length is the length of the data needed, not the event length
  1363. * which also includes the event header.
  1364. *
  1365. * Must be paired with ring_buffer_unlock_commit, unless NULL is returned.
  1366. * If NULL is returned, then nothing has been allocated or locked.
  1367. */
  1368. struct ring_buffer_event *
  1369. ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
  1370. {
  1371. struct ring_buffer_per_cpu *cpu_buffer;
  1372. struct ring_buffer_event *event;
  1373. int cpu, resched;
  1374. if (ring_buffer_flags != RB_BUFFERS_ON)
  1375. return NULL;
  1376. if (atomic_read(&buffer->record_disabled))
  1377. return NULL;
  1378. /* If we are tracing schedule, we don't want to recurse */
  1379. resched = ftrace_preempt_disable();
  1380. if (trace_recursive_lock())
  1381. goto out_nocheck;
  1382. cpu = raw_smp_processor_id();
  1383. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  1384. goto out;
  1385. cpu_buffer = buffer->buffers[cpu];
  1386. if (atomic_read(&cpu_buffer->record_disabled))
  1387. goto out;
  1388. if (length > BUF_MAX_DATA_SIZE)
  1389. goto out;
  1390. event = rb_reserve_next_event(cpu_buffer, length);
  1391. if (!event)
  1392. goto out;
  1393. /*
  1394. * Need to store resched state on this cpu.
  1395. * Only the first needs to.
  1396. */
  1397. if (preempt_count() == 1)
  1398. per_cpu(rb_need_resched, cpu) = resched;
  1399. return event;
  1400. out:
  1401. trace_recursive_unlock();
  1402. out_nocheck:
  1403. ftrace_preempt_enable(resched);
  1404. return NULL;
  1405. }
  1406. EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve);
  1407. static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
  1408. struct ring_buffer_event *event)
  1409. {
  1410. local_inc(&cpu_buffer->entries);
  1411. /*
  1412. * The event first in the commit queue updates the
  1413. * time stamp.
  1414. */
  1415. if (rb_event_is_commit(cpu_buffer, event))
  1416. cpu_buffer->write_stamp += event->time_delta;
  1417. rb_end_commit(cpu_buffer);
  1418. }
  1419. /**
  1420. * ring_buffer_unlock_commit - commit a reserved
  1421. * @buffer: The buffer to commit to
  1422. * @event: The event pointer to commit.
  1423. *
  1424. * This commits the data to the ring buffer, and releases any locks held.
  1425. *
  1426. * Must be paired with ring_buffer_lock_reserve.
  1427. */
  1428. int ring_buffer_unlock_commit(struct ring_buffer *buffer,
  1429. struct ring_buffer_event *event)
  1430. {
  1431. struct ring_buffer_per_cpu *cpu_buffer;
  1432. int cpu = raw_smp_processor_id();
  1433. cpu_buffer = buffer->buffers[cpu];
  1434. rb_commit(cpu_buffer, event);
  1435. trace_recursive_unlock();
  1436. /*
  1437. * Only the last preempt count needs to restore preemption.
  1438. */
  1439. if (preempt_count() == 1)
  1440. ftrace_preempt_enable(per_cpu(rb_need_resched, cpu));
  1441. else
  1442. preempt_enable_no_resched_notrace();
  1443. return 0;
  1444. }
  1445. EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit);
  1446. static inline void rb_event_discard(struct ring_buffer_event *event)
  1447. {
  1448. /* array[0] holds the actual length for the discarded event */
  1449. event->array[0] = rb_event_data_length(event) - RB_EVNT_HDR_SIZE;
  1450. event->type_len = RINGBUF_TYPE_PADDING;
  1451. /* time delta must be non zero */
  1452. if (!event->time_delta)
  1453. event->time_delta = 1;
  1454. }
  1455. /**
  1456. * ring_buffer_event_discard - discard any event in the ring buffer
  1457. * @event: the event to discard
  1458. *
  1459. * Sometimes a event that is in the ring buffer needs to be ignored.
  1460. * This function lets the user discard an event in the ring buffer
  1461. * and then that event will not be read later.
  1462. *
  1463. * Note, it is up to the user to be careful with this, and protect
  1464. * against races. If the user discards an event that has been consumed
  1465. * it is possible that it could corrupt the ring buffer.
  1466. */
  1467. void ring_buffer_event_discard(struct ring_buffer_event *event)
  1468. {
  1469. rb_event_discard(event);
  1470. }
  1471. EXPORT_SYMBOL_GPL(ring_buffer_event_discard);
  1472. /**
  1473. * ring_buffer_commit_discard - discard an event that has not been committed
  1474. * @buffer: the ring buffer
  1475. * @event: non committed event to discard
  1476. *
  1477. * This is similar to ring_buffer_event_discard but must only be
  1478. * performed on an event that has not been committed yet. The difference
  1479. * is that this will also try to free the event from the ring buffer
  1480. * if another event has not been added behind it.
  1481. *
  1482. * If another event has been added behind it, it will set the event
  1483. * up as discarded, and perform the commit.
  1484. *
  1485. * If this function is called, do not call ring_buffer_unlock_commit on
  1486. * the event.
  1487. */
  1488. void ring_buffer_discard_commit(struct ring_buffer *buffer,
  1489. struct ring_buffer_event *event)
  1490. {
  1491. struct ring_buffer_per_cpu *cpu_buffer;
  1492. int cpu;
  1493. /* The event is discarded regardless */
  1494. rb_event_discard(event);
  1495. cpu = smp_processor_id();
  1496. cpu_buffer = buffer->buffers[cpu];
  1497. /*
  1498. * This must only be called if the event has not been
  1499. * committed yet. Thus we can assume that preemption
  1500. * is still disabled.
  1501. */
  1502. RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing));
  1503. if (!rb_try_to_discard(cpu_buffer, event))
  1504. goto out;
  1505. /*
  1506. * The commit is still visible by the reader, so we
  1507. * must increment entries.
  1508. */
  1509. local_inc(&cpu_buffer->entries);
  1510. out:
  1511. rb_end_commit(cpu_buffer);
  1512. trace_recursive_unlock();
  1513. /*
  1514. * Only the last preempt count needs to restore preemption.
  1515. */
  1516. if (preempt_count() == 1)
  1517. ftrace_preempt_enable(per_cpu(rb_need_resched, cpu));
  1518. else
  1519. preempt_enable_no_resched_notrace();
  1520. }
  1521. EXPORT_SYMBOL_GPL(ring_buffer_discard_commit);
  1522. /**
  1523. * ring_buffer_write - write data to the buffer without reserving
  1524. * @buffer: The ring buffer to write to.
  1525. * @length: The length of the data being written (excluding the event header)
  1526. * @data: The data to write to the buffer.
  1527. *
  1528. * This is like ring_buffer_lock_reserve and ring_buffer_unlock_commit as
  1529. * one function. If you already have the data to write to the buffer, it
  1530. * may be easier to simply call this function.
  1531. *
  1532. * Note, like ring_buffer_lock_reserve, the length is the length of the data
  1533. * and not the length of the event which would hold the header.
  1534. */
  1535. int ring_buffer_write(struct ring_buffer *buffer,
  1536. unsigned long length,
  1537. void *data)
  1538. {
  1539. struct ring_buffer_per_cpu *cpu_buffer;
  1540. struct ring_buffer_event *event;
  1541. void *body;
  1542. int ret = -EBUSY;
  1543. int cpu, resched;
  1544. if (ring_buffer_flags != RB_BUFFERS_ON)
  1545. return -EBUSY;
  1546. if (atomic_read(&buffer->record_disabled))
  1547. return -EBUSY;
  1548. resched = ftrace_preempt_disable();
  1549. cpu = raw_smp_processor_id();
  1550. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  1551. goto out;
  1552. cpu_buffer = buffer->buffers[cpu];
  1553. if (atomic_read(&cpu_buffer->record_disabled))
  1554. goto out;
  1555. if (length > BUF_MAX_DATA_SIZE)
  1556. goto out;
  1557. event = rb_reserve_next_event(cpu_buffer, length);
  1558. if (!event)
  1559. goto out;
  1560. body = rb_event_data(event);
  1561. memcpy(body, data, length);
  1562. rb_commit(cpu_buffer, event);
  1563. ret = 0;
  1564. out:
  1565. ftrace_preempt_enable(resched);
  1566. return ret;
  1567. }
  1568. EXPORT_SYMBOL_GPL(ring_buffer_write);
  1569. static int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
  1570. {
  1571. struct buffer_page *reader = cpu_buffer->reader_page;
  1572. struct buffer_page *head = cpu_buffer->head_page;
  1573. struct buffer_page *commit = cpu_buffer->commit_page;
  1574. return reader->read == rb_page_commit(reader) &&
  1575. (commit == reader ||
  1576. (commit == head &&
  1577. head->read == rb_page_commit(commit)));
  1578. }
  1579. /**
  1580. * ring_buffer_record_disable - stop all writes into the buffer
  1581. * @buffer: The ring buffer to stop writes to.
  1582. *
  1583. * This prevents all writes to the buffer. Any attempt to write
  1584. * to the buffer after this will fail and return NULL.
  1585. *
  1586. * The caller should call synchronize_sched() after this.
  1587. */
  1588. void ring_buffer_record_disable(struct ring_buffer *buffer)
  1589. {
  1590. atomic_inc(&buffer->record_disabled);
  1591. }
  1592. EXPORT_SYMBOL_GPL(ring_buffer_record_disable);
  1593. /**
  1594. * ring_buffer_record_enable - enable writes to the buffer
  1595. * @buffer: The ring buffer to enable writes
  1596. *
  1597. * Note, multiple disables will need the same number of enables
  1598. * to truely enable the writing (much like preempt_disable).
  1599. */
  1600. void ring_buffer_record_enable(struct ring_buffer *buffer)
  1601. {
  1602. atomic_dec(&buffer->record_disabled);
  1603. }
  1604. EXPORT_SYMBOL_GPL(ring_buffer_record_enable);
  1605. /**
  1606. * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer
  1607. * @buffer: The ring buffer to stop writes to.
  1608. * @cpu: The CPU buffer to stop
  1609. *
  1610. * This prevents all writes to the buffer. Any attempt to write
  1611. * to the buffer after this will fail and return NULL.
  1612. *
  1613. * The caller should call synchronize_sched() after this.
  1614. */
  1615. void ring_buffer_record_disable_cpu(struct ring_buffer *buffer, int cpu)
  1616. {
  1617. struct ring_buffer_per_cpu *cpu_buffer;
  1618. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  1619. return;
  1620. cpu_buffer = buffer->buffers[cpu];
  1621. atomic_inc(&cpu_buffer->record_disabled);
  1622. }
  1623. EXPORT_SYMBOL_GPL(ring_buffer_record_disable_cpu);
  1624. /**
  1625. * ring_buffer_record_enable_cpu - enable writes to the buffer
  1626. * @buffer: The ring buffer to enable writes
  1627. * @cpu: The CPU to enable.
  1628. *
  1629. * Note, multiple disables will need the same number of enables
  1630. * to truely enable the writing (much like preempt_disable).
  1631. */
  1632. void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu)
  1633. {
  1634. struct ring_buffer_per_cpu *cpu_buffer;
  1635. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  1636. return;
  1637. cpu_buffer = buffer->buffers[cpu];
  1638. atomic_dec(&cpu_buffer->record_disabled);
  1639. }
  1640. EXPORT_SYMBOL_GPL(ring_buffer_record_enable_cpu);
  1641. /**
  1642. * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
  1643. * @buffer: The ring buffer
  1644. * @cpu: The per CPU buffer to get the entries from.
  1645. */
  1646. unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
  1647. {
  1648. struct ring_buffer_per_cpu *cpu_buffer;
  1649. unsigned long ret;
  1650. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  1651. return 0;
  1652. cpu_buffer = buffer->buffers[cpu];
  1653. ret = (local_read(&cpu_buffer->entries) - cpu_buffer->overrun)
  1654. - cpu_buffer->read;
  1655. return ret;
  1656. }
  1657. EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu);
  1658. /**
  1659. * ring_buffer_overrun_cpu - get the number of overruns in a cpu_buffer
  1660. * @buffer: The ring buffer
  1661. * @cpu: The per CPU buffer to get the number of overruns from
  1662. */
  1663. unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
  1664. {
  1665. struct ring_buffer_per_cpu *cpu_buffer;
  1666. unsigned long ret;
  1667. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  1668. return 0;
  1669. cpu_buffer = buffer->buffers[cpu];
  1670. ret = cpu_buffer->overrun;
  1671. return ret;
  1672. }
  1673. EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu);
  1674. /**
  1675. * ring_buffer_nmi_dropped_cpu - get the number of nmis that were dropped
  1676. * @buffer: The ring buffer
  1677. * @cpu: The per CPU buffer to get the number of overruns from
  1678. */
  1679. unsigned long ring_buffer_nmi_dropped_cpu(struct ring_buffer *buffer, int cpu)
  1680. {
  1681. struct ring_buffer_per_cpu *cpu_buffer;
  1682. unsigned long ret;
  1683. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  1684. return 0;
  1685. cpu_buffer = buffer->buffers[cpu];
  1686. ret = cpu_buffer->nmi_dropped;
  1687. return ret;
  1688. }
  1689. EXPORT_SYMBOL_GPL(ring_buffer_nmi_dropped_cpu);
  1690. /**
  1691. * ring_buffer_commit_overrun_cpu - get the number of overruns caused by commits
  1692. * @buffer: The ring buffer
  1693. * @cpu: The per CPU buffer to get the number of overruns from
  1694. */
  1695. unsigned long
  1696. ring_buffer_commit_overrun_cpu(struct ring_buffer *buffer, int cpu)
  1697. {
  1698. struct ring_buffer_per_cpu *cpu_buffer;
  1699. unsigned long ret;
  1700. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  1701. return 0;
  1702. cpu_buffer = buffer->buffers[cpu];
  1703. ret = cpu_buffer->commit_overrun;
  1704. return ret;
  1705. }
  1706. EXPORT_SYMBOL_GPL(ring_buffer_commit_overrun_cpu);
  1707. /**
  1708. * ring_buffer_entries - get the number of entries in a buffer
  1709. * @buffer: The ring buffer
  1710. *
  1711. * Returns the total number of entries in the ring buffer
  1712. * (all CPU entries)
  1713. */
  1714. unsigned long ring_buffer_entries(struct ring_buffer *buffer)
  1715. {
  1716. struct ring_buffer_per_cpu *cpu_buffer;
  1717. unsigned long entries = 0;
  1718. int cpu;
  1719. /* if you care about this being correct, lock the buffer */
  1720. for_each_buffer_cpu(buffer, cpu) {
  1721. cpu_buffer = buffer->buffers[cpu];
  1722. entries += (local_read(&cpu_buffer->entries) -
  1723. cpu_buffer->overrun) - cpu_buffer->read;
  1724. }
  1725. return entries;
  1726. }
  1727. EXPORT_SYMBOL_GPL(ring_buffer_entries);
  1728. /**
  1729. * ring_buffer_overrun_cpu - get the number of overruns in buffer
  1730. * @buffer: The ring buffer
  1731. *
  1732. * Returns the total number of overruns in the ring buffer
  1733. * (all CPU entries)
  1734. */
  1735. unsigned long ring_buffer_overruns(struct ring_buffer *buffer)
  1736. {
  1737. struct ring_buffer_per_cpu *cpu_buffer;
  1738. unsigned long overruns = 0;
  1739. int cpu;
  1740. /* if you care about this being correct, lock the buffer */
  1741. for_each_buffer_cpu(buffer, cpu) {
  1742. cpu_buffer = buffer->buffers[cpu];
  1743. overruns += cpu_buffer->overrun;
  1744. }
  1745. return overruns;
  1746. }
  1747. EXPORT_SYMBOL_GPL(ring_buffer_overruns);
  1748. static void rb_iter_reset(struct ring_buffer_iter *iter)
  1749. {
  1750. struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
  1751. /* Iterator usage is expected to have record disabled */
  1752. if (list_empty(&cpu_buffer->reader_page->list)) {
  1753. iter->head_page = cpu_buffer->head_page;
  1754. iter->head = cpu_buffer->head_page->read;
  1755. } else {
  1756. iter->head_page = cpu_buffer->reader_page;
  1757. iter->head = cpu_buffer->reader_page->read;
  1758. }
  1759. if (iter->head)
  1760. iter->read_stamp = cpu_buffer->read_stamp;
  1761. else
  1762. iter->read_stamp = iter->head_page->page->time_stamp;
  1763. }
  1764. /**
  1765. * ring_buffer_iter_reset - reset an iterator
  1766. * @iter: The iterator to reset
  1767. *
  1768. * Resets the iterator, so that it will start from the beginning
  1769. * again.
  1770. */
  1771. void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
  1772. {
  1773. struct ring_buffer_per_cpu *cpu_buffer;
  1774. unsigned long flags;
  1775. if (!iter)
  1776. return;
  1777. cpu_buffer = iter->cpu_buffer;
  1778. spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  1779. rb_iter_reset(iter);
  1780. spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  1781. }
  1782. EXPORT_SYMBOL_GPL(ring_buffer_iter_reset);
  1783. /**
  1784. * ring_buffer_iter_empty - check if an iterator has no more to read
  1785. * @iter: The iterator to check
  1786. */
  1787. int ring_buffer_iter_empty(struct ring_buffer_iter *iter)
  1788. {
  1789. struct ring_buffer_per_cpu *cpu_buffer;
  1790. cpu_buffer = iter->cpu_buffer;
  1791. return iter->head_page == cpu_buffer->commit_page &&
  1792. iter->head == rb_commit_index(cpu_buffer);
  1793. }
  1794. EXPORT_SYMBOL_GPL(ring_buffer_iter_empty);
  1795. static void
  1796. rb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer,
  1797. struct ring_buffer_event *event)
  1798. {
  1799. u64 delta;
  1800. switch (event->type_len) {
  1801. case RINGBUF_TYPE_PADDING:
  1802. return;
  1803. case RINGBUF_TYPE_TIME_EXTEND:
  1804. delta = event->array[0];
  1805. delta <<= TS_SHIFT;
  1806. delta += event->time_delta;
  1807. cpu_buffer->read_stamp += delta;
  1808. return;
  1809. case RINGBUF_TYPE_TIME_STAMP:
  1810. /* FIXME: not implemented */
  1811. return;
  1812. case RINGBUF_TYPE_DATA:
  1813. cpu_buffer->read_stamp += event->time_delta;
  1814. return;
  1815. default:
  1816. BUG();
  1817. }
  1818. return;
  1819. }
  1820. static void
  1821. rb_update_iter_read_stamp(struct ring_buffer_iter *iter,
  1822. struct ring_buffer_event *event)
  1823. {
  1824. u64 delta;
  1825. switch (event->type_len) {
  1826. case RINGBUF_TYPE_PADDING:
  1827. return;
  1828. case RINGBUF_TYPE_TIME_EXTEND:
  1829. delta = event->array[0];
  1830. delta <<= TS_SHIFT;
  1831. delta += event->time_delta;
  1832. iter->read_stamp += delta;
  1833. return;
  1834. case RINGBUF_TYPE_TIME_STAMP:
  1835. /* FIXME: not implemented */
  1836. return;
  1837. case RINGBUF_TYPE_DATA:
  1838. iter->read_stamp += event->time_delta;
  1839. return;
  1840. default:
  1841. BUG();
  1842. }
  1843. return;
  1844. }
  1845. static struct buffer_page *
  1846. rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
  1847. {
  1848. struct buffer_page *reader = NULL;
  1849. unsigned long flags;
  1850. int nr_loops = 0;
  1851. local_irq_save(flags);
  1852. __raw_spin_lock(&cpu_buffer->lock);
  1853. again:
  1854. /*
  1855. * This should normally only loop twice. But because the
  1856. * start of the reader inserts an empty page, it causes
  1857. * a case where we will loop three times. There should be no
  1858. * reason to loop four times (that I know of).
  1859. */
  1860. if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3)) {
  1861. reader = NULL;
  1862. goto out;
  1863. }
  1864. reader = cpu_buffer->reader_page;
  1865. /* If there's more to read, return this page */
  1866. if (cpu_buffer->reader_page->read < rb_page_size(reader))
  1867. goto out;
  1868. /* Never should we have an index greater than the size */
  1869. if (RB_WARN_ON(cpu_buffer,
  1870. cpu_buffer->reader_page->read > rb_page_size(reader)))
  1871. goto out;
  1872. /* check if we caught up to the tail */
  1873. reader = NULL;
  1874. if (cpu_buffer->commit_page == cpu_buffer->reader_page)
  1875. goto out;
  1876. /*
  1877. * Splice the empty reader page into the list around the head.
  1878. * Reset the reader page to size zero.
  1879. */
  1880. reader = cpu_buffer->head_page;
  1881. cpu_buffer->reader_page->list.next = reader->list.next;
  1882. cpu_buffer->reader_page->list.prev = reader->list.prev;
  1883. /*
  1884. * cpu_buffer->pages just needs to point to the buffer, it
  1885. * has no specific buffer page to point to. Lets move it out
  1886. * of our way so we don't accidently swap it.
  1887. */
  1888. cpu_buffer->pages = reader->list.prev;
  1889. local_set(&cpu_buffer->reader_page->write, 0);
  1890. local_set(&cpu_buffer->reader_page->entries, 0);
  1891. local_set(&cpu_buffer->reader_page->page->commit, 0);
  1892. /* Make the reader page now replace the head */
  1893. reader->list.prev->next = &cpu_buffer->reader_page->list;
  1894. reader->list.next->prev = &cpu_buffer->reader_page->list;
  1895. /*
  1896. * If the tail is on the reader, then we must set the head
  1897. * to the inserted page, otherwise we set it one before.
  1898. */
  1899. cpu_buffer->head_page = cpu_buffer->reader_page;
  1900. if (cpu_buffer->commit_page != reader)
  1901. rb_inc_page(cpu_buffer, &cpu_buffer->head_page);
  1902. /* Finally update the reader page to the new head */
  1903. cpu_buffer->reader_page = reader;
  1904. rb_reset_reader_page(cpu_buffer);
  1905. goto again;
  1906. out:
  1907. __raw_spin_unlock(&cpu_buffer->lock);
  1908. local_irq_restore(flags);
  1909. return reader;
  1910. }
  1911. static void rb_advance_reader(struct ring_buffer_per_cpu *cpu_buffer)
  1912. {
  1913. struct ring_buffer_event *event;
  1914. struct buffer_page *reader;
  1915. unsigned length;
  1916. reader = rb_get_reader_page(cpu_buffer);
  1917. /* This function should not be called when buffer is empty */
  1918. if (RB_WARN_ON(cpu_buffer, !reader))
  1919. return;
  1920. event = rb_reader_event(cpu_buffer);
  1921. if (event->type_len <= RINGBUF_TYPE_DATA_TYPE_LEN_MAX
  1922. || rb_discarded_event(event))
  1923. cpu_buffer->read++;
  1924. rb_update_read_stamp(cpu_buffer, event);
  1925. length = rb_event_length(event);
  1926. cpu_buffer->reader_page->read += length;
  1927. }
  1928. static void rb_advance_iter(struct ring_buffer_iter *iter)
  1929. {
  1930. struct ring_buffer *buffer;
  1931. struct ring_buffer_per_cpu *cpu_buffer;
  1932. struct ring_buffer_event *event;
  1933. unsigned length;
  1934. cpu_buffer = iter->cpu_buffer;
  1935. buffer = cpu_buffer->buffer;
  1936. /*
  1937. * Check if we are at the end of the buffer.
  1938. */
  1939. if (iter->head >= rb_page_size(iter->head_page)) {
  1940. /* discarded commits can make the page empty */
  1941. if (iter->head_page == cpu_buffer->commit_page)
  1942. return;
  1943. rb_inc_iter(iter);
  1944. return;
  1945. }
  1946. event = rb_iter_head_event(iter);
  1947. length = rb_event_length(event);
  1948. /*
  1949. * This should not be called to advance the header if we are
  1950. * at the tail of the buffer.
  1951. */
  1952. if (RB_WARN_ON(cpu_buffer,
  1953. (iter->head_page == cpu_buffer->commit_page) &&
  1954. (iter->head + length > rb_commit_index(cpu_buffer))))
  1955. return;
  1956. rb_update_iter_read_stamp(iter, event);
  1957. iter->head += length;
  1958. /* check for end of page padding */
  1959. if ((iter->head >= rb_page_size(iter->head_page)) &&
  1960. (iter->head_page != cpu_buffer->commit_page))
  1961. rb_advance_iter(iter);
  1962. }
  1963. static struct ring_buffer_event *
  1964. rb_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
  1965. {
  1966. struct ring_buffer_per_cpu *cpu_buffer;
  1967. struct ring_buffer_event *event;
  1968. struct buffer_page *reader;
  1969. int nr_loops = 0;
  1970. cpu_buffer = buffer->buffers[cpu];
  1971. again:
  1972. /*
  1973. * We repeat when a timestamp is encountered. It is possible
  1974. * to get multiple timestamps from an interrupt entering just
  1975. * as one timestamp is about to be written, or from discarded
  1976. * commits. The most that we can have is the number on a single page.
  1977. */
  1978. if (RB_WARN_ON(cpu_buffer, ++nr_loops > RB_TIMESTAMPS_PER_PAGE))
  1979. return NULL;
  1980. reader = rb_get_reader_page(cpu_buffer);
  1981. if (!reader)
  1982. return NULL;
  1983. event = rb_reader_event(cpu_buffer);
  1984. switch (event->type_len) {
  1985. case RINGBUF_TYPE_PADDING:
  1986. if (rb_null_event(event))
  1987. RB_WARN_ON(cpu_buffer, 1);
  1988. /*
  1989. * Because the writer could be discarding every
  1990. * event it creates (which would probably be bad)
  1991. * if we were to go back to "again" then we may never
  1992. * catch up, and will trigger the warn on, or lock
  1993. * the box. Return the padding, and we will release
  1994. * the current locks, and try again.
  1995. */
  1996. rb_advance_reader(cpu_buffer);
  1997. return event;
  1998. case RINGBUF_TYPE_TIME_EXTEND:
  1999. /* Internal data, OK to advance */
  2000. rb_advance_reader(cpu_buffer);
  2001. goto again;
  2002. case RINGBUF_TYPE_TIME_STAMP:
  2003. /* FIXME: not implemented */
  2004. rb_advance_reader(cpu_buffer);
  2005. goto again;
  2006. case RINGBUF_TYPE_DATA:
  2007. if (ts) {
  2008. *ts = cpu_buffer->read_stamp + event->time_delta;
  2009. ring_buffer_normalize_time_stamp(buffer,
  2010. cpu_buffer->cpu, ts);
  2011. }
  2012. return event;
  2013. default:
  2014. BUG();
  2015. }
  2016. return NULL;
  2017. }
  2018. EXPORT_SYMBOL_GPL(ring_buffer_peek);
  2019. static struct ring_buffer_event *
  2020. rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
  2021. {
  2022. struct ring_buffer *buffer;
  2023. struct ring_buffer_per_cpu *cpu_buffer;
  2024. struct ring_buffer_event *event;
  2025. int nr_loops = 0;
  2026. if (ring_buffer_iter_empty(iter))
  2027. return NULL;
  2028. cpu_buffer = iter->cpu_buffer;
  2029. buffer = cpu_buffer->buffer;
  2030. again:
  2031. /*
  2032. * We repeat when a timestamp is encountered.
  2033. * We can get multiple timestamps by nested interrupts or also
  2034. * if filtering is on (discarding commits). Since discarding
  2035. * commits can be frequent we can get a lot of timestamps.
  2036. * But we limit them by not adding timestamps if they begin
  2037. * at the start of a page.
  2038. */
  2039. if (RB_WARN_ON(cpu_buffer, ++nr_loops > RB_TIMESTAMPS_PER_PAGE))
  2040. return NULL;
  2041. if (rb_per_cpu_empty(cpu_buffer))
  2042. return NULL;
  2043. event = rb_iter_head_event(iter);
  2044. switch (event->type_len) {
  2045. case RINGBUF_TYPE_PADDING:
  2046. if (rb_null_event(event)) {
  2047. rb_inc_iter(iter);
  2048. goto again;
  2049. }
  2050. rb_advance_iter(iter);
  2051. return event;
  2052. case RINGBUF_TYPE_TIME_EXTEND:
  2053. /* Internal data, OK to advance */
  2054. rb_advance_iter(iter);
  2055. goto again;
  2056. case RINGBUF_TYPE_TIME_STAMP:
  2057. /* FIXME: not implemented */
  2058. rb_advance_iter(iter);
  2059. goto again;
  2060. case RINGBUF_TYPE_DATA:
  2061. if (ts) {
  2062. *ts = iter->read_stamp + event->time_delta;
  2063. ring_buffer_normalize_time_stamp(buffer,
  2064. cpu_buffer->cpu, ts);
  2065. }
  2066. return event;
  2067. default:
  2068. BUG();
  2069. }
  2070. return NULL;
  2071. }
  2072. EXPORT_SYMBOL_GPL(ring_buffer_iter_peek);
  2073. static inline int rb_ok_to_lock(void)
  2074. {
  2075. /*
  2076. * If an NMI die dumps out the content of the ring buffer
  2077. * do not grab locks. We also permanently disable the ring
  2078. * buffer too. A one time deal is all you get from reading
  2079. * the ring buffer from an NMI.
  2080. */
  2081. if (likely(!in_nmi() && !oops_in_progress))
  2082. return 1;
  2083. tracing_off_permanent();
  2084. return 0;
  2085. }
  2086. /**
  2087. * ring_buffer_peek - peek at the next event to be read
  2088. * @buffer: The ring buffer to read
  2089. * @cpu: The cpu to peak at
  2090. * @ts: The timestamp counter of this event.
  2091. *
  2092. * This will return the event that will be read next, but does
  2093. * not consume the data.
  2094. */
  2095. struct ring_buffer_event *
  2096. ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
  2097. {
  2098. struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
  2099. struct ring_buffer_event *event;
  2100. unsigned long flags;
  2101. int dolock;
  2102. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2103. return NULL;
  2104. dolock = rb_ok_to_lock();
  2105. again:
  2106. local_irq_save(flags);
  2107. if (dolock)
  2108. spin_lock(&cpu_buffer->reader_lock);
  2109. event = rb_buffer_peek(buffer, cpu, ts);
  2110. if (dolock)
  2111. spin_unlock(&cpu_buffer->reader_lock);
  2112. local_irq_restore(flags);
  2113. if (event && event->type_len == RINGBUF_TYPE_PADDING) {
  2114. cpu_relax();
  2115. goto again;
  2116. }
  2117. return event;
  2118. }
  2119. /**
  2120. * ring_buffer_iter_peek - peek at the next event to be read
  2121. * @iter: The ring buffer iterator
  2122. * @ts: The timestamp counter of this event.
  2123. *
  2124. * This will return the event that will be read next, but does
  2125. * not increment the iterator.
  2126. */
  2127. struct ring_buffer_event *
  2128. ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
  2129. {
  2130. struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
  2131. struct ring_buffer_event *event;
  2132. unsigned long flags;
  2133. again:
  2134. spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  2135. event = rb_iter_peek(iter, ts);
  2136. spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  2137. if (event && event->type_len == RINGBUF_TYPE_PADDING) {
  2138. cpu_relax();
  2139. goto again;
  2140. }
  2141. return event;
  2142. }
  2143. /**
  2144. * ring_buffer_consume - return an event and consume it
  2145. * @buffer: The ring buffer to get the next event from
  2146. *
  2147. * Returns the next event in the ring buffer, and that event is consumed.
  2148. * Meaning, that sequential reads will keep returning a different event,
  2149. * and eventually empty the ring buffer if the producer is slower.
  2150. */
  2151. struct ring_buffer_event *
  2152. ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
  2153. {
  2154. struct ring_buffer_per_cpu *cpu_buffer;
  2155. struct ring_buffer_event *event = NULL;
  2156. unsigned long flags;
  2157. int dolock;
  2158. dolock = rb_ok_to_lock();
  2159. again:
  2160. /* might be called in atomic */
  2161. preempt_disable();
  2162. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2163. goto out;
  2164. cpu_buffer = buffer->buffers[cpu];
  2165. local_irq_save(flags);
  2166. if (dolock)
  2167. spin_lock(&cpu_buffer->reader_lock);
  2168. event = rb_buffer_peek(buffer, cpu, ts);
  2169. if (!event)
  2170. goto out_unlock;
  2171. rb_advance_reader(cpu_buffer);
  2172. out_unlock:
  2173. if (dolock)
  2174. spin_unlock(&cpu_buffer->reader_lock);
  2175. local_irq_restore(flags);
  2176. out:
  2177. preempt_enable();
  2178. if (event && event->type_len == RINGBUF_TYPE_PADDING) {
  2179. cpu_relax();
  2180. goto again;
  2181. }
  2182. return event;
  2183. }
  2184. EXPORT_SYMBOL_GPL(ring_buffer_consume);
  2185. /**
  2186. * ring_buffer_read_start - start a non consuming read of the buffer
  2187. * @buffer: The ring buffer to read from
  2188. * @cpu: The cpu buffer to iterate over
  2189. *
  2190. * This starts up an iteration through the buffer. It also disables
  2191. * the recording to the buffer until the reading is finished.
  2192. * This prevents the reading from being corrupted. This is not
  2193. * a consuming read, so a producer is not expected.
  2194. *
  2195. * Must be paired with ring_buffer_finish.
  2196. */
  2197. struct ring_buffer_iter *
  2198. ring_buffer_read_start(struct ring_buffer *buffer, int cpu)
  2199. {
  2200. struct ring_buffer_per_cpu *cpu_buffer;
  2201. struct ring_buffer_iter *iter;
  2202. unsigned long flags;
  2203. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2204. return NULL;
  2205. iter = kmalloc(sizeof(*iter), GFP_KERNEL);
  2206. if (!iter)
  2207. return NULL;
  2208. cpu_buffer = buffer->buffers[cpu];
  2209. iter->cpu_buffer = cpu_buffer;
  2210. atomic_inc(&cpu_buffer->record_disabled);
  2211. synchronize_sched();
  2212. spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  2213. __raw_spin_lock(&cpu_buffer->lock);
  2214. rb_iter_reset(iter);
  2215. __raw_spin_unlock(&cpu_buffer->lock);
  2216. spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  2217. return iter;
  2218. }
  2219. EXPORT_SYMBOL_GPL(ring_buffer_read_start);
  2220. /**
  2221. * ring_buffer_finish - finish reading the iterator of the buffer
  2222. * @iter: The iterator retrieved by ring_buffer_start
  2223. *
  2224. * This re-enables the recording to the buffer, and frees the
  2225. * iterator.
  2226. */
  2227. void
  2228. ring_buffer_read_finish(struct ring_buffer_iter *iter)
  2229. {
  2230. struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
  2231. atomic_dec(&cpu_buffer->record_disabled);
  2232. kfree(iter);
  2233. }
  2234. EXPORT_SYMBOL_GPL(ring_buffer_read_finish);
  2235. /**
  2236. * ring_buffer_read - read the next item in the ring buffer by the iterator
  2237. * @iter: The ring buffer iterator
  2238. * @ts: The time stamp of the event read.
  2239. *
  2240. * This reads the next event in the ring buffer and increments the iterator.
  2241. */
  2242. struct ring_buffer_event *
  2243. ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
  2244. {
  2245. struct ring_buffer_event *event;
  2246. struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
  2247. unsigned long flags;
  2248. again:
  2249. spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  2250. event = rb_iter_peek(iter, ts);
  2251. if (!event)
  2252. goto out;
  2253. rb_advance_iter(iter);
  2254. out:
  2255. spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  2256. if (event && event->type_len == RINGBUF_TYPE_PADDING) {
  2257. cpu_relax();
  2258. goto again;
  2259. }
  2260. return event;
  2261. }
  2262. EXPORT_SYMBOL_GPL(ring_buffer_read);
  2263. /**
  2264. * ring_buffer_size - return the size of the ring buffer (in bytes)
  2265. * @buffer: The ring buffer.
  2266. */
  2267. unsigned long ring_buffer_size(struct ring_buffer *buffer)
  2268. {
  2269. return BUF_PAGE_SIZE * buffer->pages;
  2270. }
  2271. EXPORT_SYMBOL_GPL(ring_buffer_size);
  2272. static void
  2273. rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
  2274. {
  2275. cpu_buffer->head_page
  2276. = list_entry(cpu_buffer->pages, struct buffer_page, list);
  2277. local_set(&cpu_buffer->head_page->write, 0);
  2278. local_set(&cpu_buffer->head_page->entries, 0);
  2279. local_set(&cpu_buffer->head_page->page->commit, 0);
  2280. cpu_buffer->head_page->read = 0;
  2281. cpu_buffer->tail_page = cpu_buffer->head_page;
  2282. cpu_buffer->commit_page = cpu_buffer->head_page;
  2283. INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
  2284. local_set(&cpu_buffer->reader_page->write, 0);
  2285. local_set(&cpu_buffer->reader_page->entries, 0);
  2286. local_set(&cpu_buffer->reader_page->page->commit, 0);
  2287. cpu_buffer->reader_page->read = 0;
  2288. cpu_buffer->nmi_dropped = 0;
  2289. cpu_buffer->commit_overrun = 0;
  2290. cpu_buffer->overrun = 0;
  2291. cpu_buffer->read = 0;
  2292. local_set(&cpu_buffer->entries, 0);
  2293. local_set(&cpu_buffer->committing, 0);
  2294. local_set(&cpu_buffer->commits, 0);
  2295. cpu_buffer->write_stamp = 0;
  2296. cpu_buffer->read_stamp = 0;
  2297. }
  2298. /**
  2299. * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
  2300. * @buffer: The ring buffer to reset a per cpu buffer of
  2301. * @cpu: The CPU buffer to be reset
  2302. */
  2303. void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
  2304. {
  2305. struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
  2306. unsigned long flags;
  2307. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2308. return;
  2309. atomic_inc(&cpu_buffer->record_disabled);
  2310. spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  2311. __raw_spin_lock(&cpu_buffer->lock);
  2312. rb_reset_cpu(cpu_buffer);
  2313. __raw_spin_unlock(&cpu_buffer->lock);
  2314. spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  2315. atomic_dec(&cpu_buffer->record_disabled);
  2316. }
  2317. EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);
  2318. /**
  2319. * ring_buffer_reset - reset a ring buffer
  2320. * @buffer: The ring buffer to reset all cpu buffers
  2321. */
  2322. void ring_buffer_reset(struct ring_buffer *buffer)
  2323. {
  2324. int cpu;
  2325. for_each_buffer_cpu(buffer, cpu)
  2326. ring_buffer_reset_cpu(buffer, cpu);
  2327. }
  2328. EXPORT_SYMBOL_GPL(ring_buffer_reset);
  2329. /**
  2330. * rind_buffer_empty - is the ring buffer empty?
  2331. * @buffer: The ring buffer to test
  2332. */
  2333. int ring_buffer_empty(struct ring_buffer *buffer)
  2334. {
  2335. struct ring_buffer_per_cpu *cpu_buffer;
  2336. unsigned long flags;
  2337. int dolock;
  2338. int cpu;
  2339. int ret;
  2340. dolock = rb_ok_to_lock();
  2341. /* yes this is racy, but if you don't like the race, lock the buffer */
  2342. for_each_buffer_cpu(buffer, cpu) {
  2343. cpu_buffer = buffer->buffers[cpu];
  2344. local_irq_save(flags);
  2345. if (dolock)
  2346. spin_lock(&cpu_buffer->reader_lock);
  2347. ret = rb_per_cpu_empty(cpu_buffer);
  2348. if (dolock)
  2349. spin_unlock(&cpu_buffer->reader_lock);
  2350. local_irq_restore(flags);
  2351. if (!ret)
  2352. return 0;
  2353. }
  2354. return 1;
  2355. }
  2356. EXPORT_SYMBOL_GPL(ring_buffer_empty);
  2357. /**
  2358. * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
  2359. * @buffer: The ring buffer
  2360. * @cpu: The CPU buffer to test
  2361. */
  2362. int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
  2363. {
  2364. struct ring_buffer_per_cpu *cpu_buffer;
  2365. unsigned long flags;
  2366. int dolock;
  2367. int ret;
  2368. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2369. return 1;
  2370. dolock = rb_ok_to_lock();
  2371. cpu_buffer = buffer->buffers[cpu];
  2372. local_irq_save(flags);
  2373. if (dolock)
  2374. spin_lock(&cpu_buffer->reader_lock);
  2375. ret = rb_per_cpu_empty(cpu_buffer);
  2376. if (dolock)
  2377. spin_unlock(&cpu_buffer->reader_lock);
  2378. local_irq_restore(flags);
  2379. return ret;
  2380. }
  2381. EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu);
  2382. /**
  2383. * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
  2384. * @buffer_a: One buffer to swap with
  2385. * @buffer_b: The other buffer to swap with
  2386. *
  2387. * This function is useful for tracers that want to take a "snapshot"
  2388. * of a CPU buffer and has another back up buffer lying around.
  2389. * it is expected that the tracer handles the cpu buffer not being
  2390. * used at the moment.
  2391. */
  2392. int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
  2393. struct ring_buffer *buffer_b, int cpu)
  2394. {
  2395. struct ring_buffer_per_cpu *cpu_buffer_a;
  2396. struct ring_buffer_per_cpu *cpu_buffer_b;
  2397. int ret = -EINVAL;
  2398. if (!cpumask_test_cpu(cpu, buffer_a->cpumask) ||
  2399. !cpumask_test_cpu(cpu, buffer_b->cpumask))
  2400. goto out;
  2401. /* At least make sure the two buffers are somewhat the same */
  2402. if (buffer_a->pages != buffer_b->pages)
  2403. goto out;
  2404. ret = -EAGAIN;
  2405. if (ring_buffer_flags != RB_BUFFERS_ON)
  2406. goto out;
  2407. if (atomic_read(&buffer_a->record_disabled))
  2408. goto out;
  2409. if (atomic_read(&buffer_b->record_disabled))
  2410. goto out;
  2411. cpu_buffer_a = buffer_a->buffers[cpu];
  2412. cpu_buffer_b = buffer_b->buffers[cpu];
  2413. if (atomic_read(&cpu_buffer_a->record_disabled))
  2414. goto out;
  2415. if (atomic_read(&cpu_buffer_b->record_disabled))
  2416. goto out;
  2417. /*
  2418. * We can't do a synchronize_sched here because this
  2419. * function can be called in atomic context.
  2420. * Normally this will be called from the same CPU as cpu.
  2421. * If not it's up to the caller to protect this.
  2422. */
  2423. atomic_inc(&cpu_buffer_a->record_disabled);
  2424. atomic_inc(&cpu_buffer_b->record_disabled);
  2425. buffer_a->buffers[cpu] = cpu_buffer_b;
  2426. buffer_b->buffers[cpu] = cpu_buffer_a;
  2427. cpu_buffer_b->buffer = buffer_a;
  2428. cpu_buffer_a->buffer = buffer_b;
  2429. atomic_dec(&cpu_buffer_a->record_disabled);
  2430. atomic_dec(&cpu_buffer_b->record_disabled);
  2431. ret = 0;
  2432. out:
  2433. return ret;
  2434. }
  2435. EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
  2436. /**
  2437. * ring_buffer_alloc_read_page - allocate a page to read from buffer
  2438. * @buffer: the buffer to allocate for.
  2439. *
  2440. * This function is used in conjunction with ring_buffer_read_page.
  2441. * When reading a full page from the ring buffer, these functions
  2442. * can be used to speed up the process. The calling function should
  2443. * allocate a few pages first with this function. Then when it
  2444. * needs to get pages from the ring buffer, it passes the result
  2445. * of this function into ring_buffer_read_page, which will swap
  2446. * the page that was allocated, with the read page of the buffer.
  2447. *
  2448. * Returns:
  2449. * The page allocated, or NULL on error.
  2450. */
  2451. void *ring_buffer_alloc_read_page(struct ring_buffer *buffer)
  2452. {
  2453. struct buffer_data_page *bpage;
  2454. unsigned long addr;
  2455. addr = __get_free_page(GFP_KERNEL);
  2456. if (!addr)
  2457. return NULL;
  2458. bpage = (void *)addr;
  2459. rb_init_page(bpage);
  2460. return bpage;
  2461. }
  2462. EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page);
  2463. /**
  2464. * ring_buffer_free_read_page - free an allocated read page
  2465. * @buffer: the buffer the page was allocate for
  2466. * @data: the page to free
  2467. *
  2468. * Free a page allocated from ring_buffer_alloc_read_page.
  2469. */
  2470. void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data)
  2471. {
  2472. free_page((unsigned long)data);
  2473. }
  2474. EXPORT_SYMBOL_GPL(ring_buffer_free_read_page);
  2475. /**
  2476. * ring_buffer_read_page - extract a page from the ring buffer
  2477. * @buffer: buffer to extract from
  2478. * @data_page: the page to use allocated from ring_buffer_alloc_read_page
  2479. * @len: amount to extract
  2480. * @cpu: the cpu of the buffer to extract
  2481. * @full: should the extraction only happen when the page is full.
  2482. *
  2483. * This function will pull out a page from the ring buffer and consume it.
  2484. * @data_page must be the address of the variable that was returned
  2485. * from ring_buffer_alloc_read_page. This is because the page might be used
  2486. * to swap with a page in the ring buffer.
  2487. *
  2488. * for example:
  2489. * rpage = ring_buffer_alloc_read_page(buffer);
  2490. * if (!rpage)
  2491. * return error;
  2492. * ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0);
  2493. * if (ret >= 0)
  2494. * process_page(rpage, ret);
  2495. *
  2496. * When @full is set, the function will not return true unless
  2497. * the writer is off the reader page.
  2498. *
  2499. * Note: it is up to the calling functions to handle sleeps and wakeups.
  2500. * The ring buffer can be used anywhere in the kernel and can not
  2501. * blindly call wake_up. The layer that uses the ring buffer must be
  2502. * responsible for that.
  2503. *
  2504. * Returns:
  2505. * >=0 if data has been transferred, returns the offset of consumed data.
  2506. * <0 if no data has been transferred.
  2507. */
  2508. int ring_buffer_read_page(struct ring_buffer *buffer,
  2509. void **data_page, size_t len, int cpu, int full)
  2510. {
  2511. struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
  2512. struct ring_buffer_event *event;
  2513. struct buffer_data_page *bpage;
  2514. struct buffer_page *reader;
  2515. unsigned long flags;
  2516. unsigned int commit;
  2517. unsigned int read;
  2518. u64 save_timestamp;
  2519. int ret = -1;
  2520. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2521. goto out;
  2522. /*
  2523. * If len is not big enough to hold the page header, then
  2524. * we can not copy anything.
  2525. */
  2526. if (len <= BUF_PAGE_HDR_SIZE)
  2527. goto out;
  2528. len -= BUF_PAGE_HDR_SIZE;
  2529. if (!data_page)
  2530. goto out;
  2531. bpage = *data_page;
  2532. if (!bpage)
  2533. goto out;
  2534. spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  2535. reader = rb_get_reader_page(cpu_buffer);
  2536. if (!reader)
  2537. goto out_unlock;
  2538. event = rb_reader_event(cpu_buffer);
  2539. read = reader->read;
  2540. commit = rb_page_commit(reader);
  2541. /*
  2542. * If this page has been partially read or
  2543. * if len is not big enough to read the rest of the page or
  2544. * a writer is still on the page, then
  2545. * we must copy the data from the page to the buffer.
  2546. * Otherwise, we can simply swap the page with the one passed in.
  2547. */
  2548. if (read || (len < (commit - read)) ||
  2549. cpu_buffer->reader_page == cpu_buffer->commit_page) {
  2550. struct buffer_data_page *rpage = cpu_buffer->reader_page->page;
  2551. unsigned int rpos = read;
  2552. unsigned int pos = 0;
  2553. unsigned int size;
  2554. if (full)
  2555. goto out_unlock;
  2556. if (len > (commit - read))
  2557. len = (commit - read);
  2558. size = rb_event_length(event);
  2559. if (len < size)
  2560. goto out_unlock;
  2561. /* save the current timestamp, since the user will need it */
  2562. save_timestamp = cpu_buffer->read_stamp;
  2563. /* Need to copy one event at a time */
  2564. do {
  2565. memcpy(bpage->data + pos, rpage->data + rpos, size);
  2566. len -= size;
  2567. rb_advance_reader(cpu_buffer);
  2568. rpos = reader->read;
  2569. pos += size;
  2570. event = rb_reader_event(cpu_buffer);
  2571. size = rb_event_length(event);
  2572. } while (len > size);
  2573. /* update bpage */
  2574. local_set(&bpage->commit, pos);
  2575. bpage->time_stamp = save_timestamp;
  2576. /* we copied everything to the beginning */
  2577. read = 0;
  2578. } else {
  2579. /* update the entry counter */
  2580. cpu_buffer->read += local_read(&reader->entries);
  2581. /* swap the pages */
  2582. rb_init_page(bpage);
  2583. bpage = reader->page;
  2584. reader->page = *data_page;
  2585. local_set(&reader->write, 0);
  2586. local_set(&reader->entries, 0);
  2587. reader->read = 0;
  2588. *data_page = bpage;
  2589. }
  2590. ret = read;
  2591. out_unlock:
  2592. spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  2593. out:
  2594. return ret;
  2595. }
  2596. EXPORT_SYMBOL_GPL(ring_buffer_read_page);
  2597. #ifdef CONFIG_TRACING
  2598. static ssize_t
  2599. rb_simple_read(struct file *filp, char __user *ubuf,
  2600. size_t cnt, loff_t *ppos)
  2601. {
  2602. unsigned long *p = filp->private_data;
  2603. char buf[64];
  2604. int r;
  2605. if (test_bit(RB_BUFFERS_DISABLED_BIT, p))
  2606. r = sprintf(buf, "permanently disabled\n");
  2607. else
  2608. r = sprintf(buf, "%d\n", test_bit(RB_BUFFERS_ON_BIT, p));
  2609. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  2610. }
  2611. static ssize_t
  2612. rb_simple_write(struct file *filp, const char __user *ubuf,
  2613. size_t cnt, loff_t *ppos)
  2614. {
  2615. unsigned long *p = filp->private_data;
  2616. char buf[64];
  2617. unsigned long val;
  2618. int ret;
  2619. if (cnt >= sizeof(buf))
  2620. return -EINVAL;
  2621. if (copy_from_user(&buf, ubuf, cnt))
  2622. return -EFAULT;
  2623. buf[cnt] = 0;
  2624. ret = strict_strtoul(buf, 10, &val);
  2625. if (ret < 0)
  2626. return ret;
  2627. if (val)
  2628. set_bit(RB_BUFFERS_ON_BIT, p);
  2629. else
  2630. clear_bit(RB_BUFFERS_ON_BIT, p);
  2631. (*ppos)++;
  2632. return cnt;
  2633. }
  2634. static const struct file_operations rb_simple_fops = {
  2635. .open = tracing_open_generic,
  2636. .read = rb_simple_read,
  2637. .write = rb_simple_write,
  2638. };
  2639. static __init int rb_init_debugfs(void)
  2640. {
  2641. struct dentry *d_tracer;
  2642. d_tracer = tracing_init_dentry();
  2643. trace_create_file("tracing_on", 0644, d_tracer,
  2644. &ring_buffer_flags, &rb_simple_fops);
  2645. return 0;
  2646. }
  2647. fs_initcall(rb_init_debugfs);
  2648. #endif
  2649. #ifdef CONFIG_HOTPLUG_CPU
  2650. static int rb_cpu_notify(struct notifier_block *self,
  2651. unsigned long action, void *hcpu)
  2652. {
  2653. struct ring_buffer *buffer =
  2654. container_of(self, struct ring_buffer, cpu_notify);
  2655. long cpu = (long)hcpu;
  2656. switch (action) {
  2657. case CPU_UP_PREPARE:
  2658. case CPU_UP_PREPARE_FROZEN:
  2659. if (cpumask_test_cpu(cpu, buffer->cpumask))
  2660. return NOTIFY_OK;
  2661. buffer->buffers[cpu] =
  2662. rb_allocate_cpu_buffer(buffer, cpu);
  2663. if (!buffer->buffers[cpu]) {
  2664. WARN(1, "failed to allocate ring buffer on CPU %ld\n",
  2665. cpu);
  2666. return NOTIFY_OK;
  2667. }
  2668. smp_wmb();
  2669. cpumask_set_cpu(cpu, buffer->cpumask);
  2670. break;
  2671. case CPU_DOWN_PREPARE:
  2672. case CPU_DOWN_PREPARE_FROZEN:
  2673. /*
  2674. * Do nothing.
  2675. * If we were to free the buffer, then the user would
  2676. * lose any trace that was in the buffer.
  2677. */
  2678. break;
  2679. default:
  2680. break;
  2681. }
  2682. return NOTIFY_OK;
  2683. }
  2684. #endif