ring_buffer.c 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016
  1. /*
  2. * Generic ring buffer
  3. *
  4. * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
  5. */
  6. #include <linux/ring_buffer.h>
  7. #include <linux/spinlock.h>
  8. #include <linux/debugfs.h>
  9. #include <linux/uaccess.h>
  10. #include <linux/module.h>
  11. #include <linux/percpu.h>
  12. #include <linux/mutex.h>
  13. #include <linux/sched.h> /* used for sched_clock() (for now) */
  14. #include <linux/init.h>
  15. #include <linux/hash.h>
  16. #include <linux/list.h>
  17. #include <linux/fs.h>
  18. /* Up this if you want to test the TIME_EXTENTS and normalization */
  19. #define DEBUG_SHIFT 0
  20. /* FIXME!!! */
  21. u64 ring_buffer_time_stamp(int cpu)
  22. {
  23. /* shift to debug/test normalization and TIME_EXTENTS */
  24. return sched_clock() << DEBUG_SHIFT;
  25. }
  26. void ring_buffer_normalize_time_stamp(int cpu, u64 *ts)
  27. {
  28. /* Just stupid testing the normalize function and deltas */
  29. *ts >>= DEBUG_SHIFT;
  30. }
  31. #define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event))
  32. #define RB_ALIGNMENT_SHIFT 2
  33. #define RB_ALIGNMENT (1 << RB_ALIGNMENT_SHIFT)
  34. #define RB_MAX_SMALL_DATA 28
  35. enum {
  36. RB_LEN_TIME_EXTEND = 8,
  37. RB_LEN_TIME_STAMP = 16,
  38. };
  39. /* inline for ring buffer fast paths */
  40. static inline unsigned
  41. rb_event_length(struct ring_buffer_event *event)
  42. {
  43. unsigned length;
  44. switch (event->type) {
  45. case RINGBUF_TYPE_PADDING:
  46. /* undefined */
  47. return -1;
  48. case RINGBUF_TYPE_TIME_EXTEND:
  49. return RB_LEN_TIME_EXTEND;
  50. case RINGBUF_TYPE_TIME_STAMP:
  51. return RB_LEN_TIME_STAMP;
  52. case RINGBUF_TYPE_DATA:
  53. if (event->len)
  54. length = event->len << RB_ALIGNMENT_SHIFT;
  55. else
  56. length = event->array[0];
  57. return length + RB_EVNT_HDR_SIZE;
  58. default:
  59. BUG();
  60. }
  61. /* not hit */
  62. return 0;
  63. }
  64. /**
  65. * ring_buffer_event_length - return the length of the event
  66. * @event: the event to get the length of
  67. */
  68. unsigned ring_buffer_event_length(struct ring_buffer_event *event)
  69. {
  70. return rb_event_length(event);
  71. }
  72. /* inline for ring buffer fast paths */
  73. static inline void *
  74. rb_event_data(struct ring_buffer_event *event)
  75. {
  76. BUG_ON(event->type != RINGBUF_TYPE_DATA);
  77. /* If length is in len field, then array[0] has the data */
  78. if (event->len)
  79. return (void *)&event->array[0];
  80. /* Otherwise length is in array[0] and array[1] has the data */
  81. return (void *)&event->array[1];
  82. }
  83. /**
  84. * ring_buffer_event_data - return the data of the event
  85. * @event: the event to get the data from
  86. */
  87. void *ring_buffer_event_data(struct ring_buffer_event *event)
  88. {
  89. return rb_event_data(event);
  90. }
  91. #define for_each_buffer_cpu(buffer, cpu) \
  92. for_each_cpu_mask(cpu, buffer->cpumask)
  93. #define TS_SHIFT 27
  94. #define TS_MASK ((1ULL << TS_SHIFT) - 1)
  95. #define TS_DELTA_TEST (~TS_MASK)
  96. /*
  97. * This hack stolen from mm/slob.c.
  98. * We can store per page timing information in the page frame of the page.
  99. * Thanks to Peter Zijlstra for suggesting this idea.
  100. */
  101. struct buffer_page {
  102. u64 time_stamp; /* page time stamp */
  103. local_t write; /* index for next write */
  104. local_t commit; /* write commited index */
  105. unsigned read; /* index for next read */
  106. struct list_head list; /* list of free pages */
  107. void *page; /* Actual data page */
  108. };
  109. /*
  110. * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing
  111. * this issue out.
  112. */
  113. static inline void free_buffer_page(struct buffer_page *bpage)
  114. {
  115. if (bpage->page)
  116. free_page((unsigned long)bpage->page);
  117. kfree(bpage);
  118. }
  119. /*
  120. * We need to fit the time_stamp delta into 27 bits.
  121. */
  122. static inline int test_time_stamp(u64 delta)
  123. {
  124. if (delta & TS_DELTA_TEST)
  125. return 1;
  126. return 0;
  127. }
  128. #define BUF_PAGE_SIZE PAGE_SIZE
  129. /*
  130. * head_page == tail_page && head == tail then buffer is empty.
  131. */
  132. struct ring_buffer_per_cpu {
  133. int cpu;
  134. struct ring_buffer *buffer;
  135. spinlock_t lock;
  136. struct lock_class_key lock_key;
  137. struct list_head pages;
  138. struct buffer_page *head_page; /* read from head */
  139. struct buffer_page *tail_page; /* write to tail */
  140. struct buffer_page *commit_page; /* commited pages */
  141. struct buffer_page *reader_page;
  142. unsigned long overrun;
  143. unsigned long entries;
  144. u64 write_stamp;
  145. u64 read_stamp;
  146. atomic_t record_disabled;
  147. };
  148. struct ring_buffer {
  149. unsigned long size;
  150. unsigned pages;
  151. unsigned flags;
  152. int cpus;
  153. cpumask_t cpumask;
  154. atomic_t record_disabled;
  155. struct mutex mutex;
  156. struct ring_buffer_per_cpu **buffers;
  157. };
  158. struct ring_buffer_iter {
  159. struct ring_buffer_per_cpu *cpu_buffer;
  160. unsigned long head;
  161. struct buffer_page *head_page;
  162. u64 read_stamp;
  163. };
  164. #define RB_WARN_ON(buffer, cond) \
  165. do { \
  166. if (unlikely(cond)) { \
  167. atomic_inc(&buffer->record_disabled); \
  168. WARN_ON(1); \
  169. } \
  170. } while (0)
  171. #define RB_WARN_ON_RET(buffer, cond) \
  172. do { \
  173. if (unlikely(cond)) { \
  174. atomic_inc(&buffer->record_disabled); \
  175. WARN_ON(1); \
  176. return -1; \
  177. } \
  178. } while (0)
  179. #define RB_WARN_ON_ONCE(buffer, cond) \
  180. do { \
  181. static int once; \
  182. if (unlikely(cond) && !once) { \
  183. once++; \
  184. atomic_inc(&buffer->record_disabled); \
  185. WARN_ON(1); \
  186. } \
  187. } while (0)
  188. /**
  189. * check_pages - integrity check of buffer pages
  190. * @cpu_buffer: CPU buffer with pages to test
  191. *
  192. * As a safty measure we check to make sure the data pages have not
  193. * been corrupted.
  194. */
  195. static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
  196. {
  197. struct list_head *head = &cpu_buffer->pages;
  198. struct buffer_page *page, *tmp;
  199. RB_WARN_ON_RET(cpu_buffer, head->next->prev != head);
  200. RB_WARN_ON_RET(cpu_buffer, head->prev->next != head);
  201. list_for_each_entry_safe(page, tmp, head, list) {
  202. RB_WARN_ON_RET(cpu_buffer,
  203. page->list.next->prev != &page->list);
  204. RB_WARN_ON_RET(cpu_buffer,
  205. page->list.prev->next != &page->list);
  206. }
  207. return 0;
  208. }
  209. static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
  210. unsigned nr_pages)
  211. {
  212. struct list_head *head = &cpu_buffer->pages;
  213. struct buffer_page *page, *tmp;
  214. unsigned long addr;
  215. LIST_HEAD(pages);
  216. unsigned i;
  217. for (i = 0; i < nr_pages; i++) {
  218. page = kzalloc_node(ALIGN(sizeof(*page), cache_line_size()),
  219. GFP_KERNEL, cpu_to_node(cpu_buffer->cpu));
  220. if (!page)
  221. goto free_pages;
  222. list_add(&page->list, &pages);
  223. addr = __get_free_page(GFP_KERNEL);
  224. if (!addr)
  225. goto free_pages;
  226. page->page = (void *)addr;
  227. }
  228. list_splice(&pages, head);
  229. rb_check_pages(cpu_buffer);
  230. return 0;
  231. free_pages:
  232. list_for_each_entry_safe(page, tmp, &pages, list) {
  233. list_del_init(&page->list);
  234. free_buffer_page(page);
  235. }
  236. return -ENOMEM;
  237. }
  238. static struct ring_buffer_per_cpu *
  239. rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
  240. {
  241. struct ring_buffer_per_cpu *cpu_buffer;
  242. struct buffer_page *page;
  243. unsigned long addr;
  244. int ret;
  245. cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()),
  246. GFP_KERNEL, cpu_to_node(cpu));
  247. if (!cpu_buffer)
  248. return NULL;
  249. cpu_buffer->cpu = cpu;
  250. cpu_buffer->buffer = buffer;
  251. spin_lock_init(&cpu_buffer->lock);
  252. INIT_LIST_HEAD(&cpu_buffer->pages);
  253. page = kzalloc_node(ALIGN(sizeof(*page), cache_line_size()),
  254. GFP_KERNEL, cpu_to_node(cpu));
  255. if (!page)
  256. goto fail_free_buffer;
  257. cpu_buffer->reader_page = page;
  258. addr = __get_free_page(GFP_KERNEL);
  259. if (!addr)
  260. goto fail_free_reader;
  261. page->page = (void *)addr;
  262. INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
  263. ret = rb_allocate_pages(cpu_buffer, buffer->pages);
  264. if (ret < 0)
  265. goto fail_free_reader;
  266. cpu_buffer->head_page
  267. = list_entry(cpu_buffer->pages.next, struct buffer_page, list);
  268. cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;
  269. return cpu_buffer;
  270. fail_free_reader:
  271. free_buffer_page(cpu_buffer->reader_page);
  272. fail_free_buffer:
  273. kfree(cpu_buffer);
  274. return NULL;
  275. }
  276. static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
  277. {
  278. struct list_head *head = &cpu_buffer->pages;
  279. struct buffer_page *page, *tmp;
  280. list_del_init(&cpu_buffer->reader_page->list);
  281. free_buffer_page(cpu_buffer->reader_page);
  282. list_for_each_entry_safe(page, tmp, head, list) {
  283. list_del_init(&page->list);
  284. free_buffer_page(page);
  285. }
  286. kfree(cpu_buffer);
  287. }
  288. /*
  289. * Causes compile errors if the struct buffer_page gets bigger
  290. * than the struct page.
  291. */
  292. extern int ring_buffer_page_too_big(void);
  293. /**
  294. * ring_buffer_alloc - allocate a new ring_buffer
  295. * @size: the size in bytes that is needed.
  296. * @flags: attributes to set for the ring buffer.
  297. *
  298. * Currently the only flag that is available is the RB_FL_OVERWRITE
  299. * flag. This flag means that the buffer will overwrite old data
  300. * when the buffer wraps. If this flag is not set, the buffer will
  301. * drop data when the tail hits the head.
  302. */
  303. struct ring_buffer *ring_buffer_alloc(unsigned long size, unsigned flags)
  304. {
  305. struct ring_buffer *buffer;
  306. int bsize;
  307. int cpu;
  308. /* Paranoid! Optimizes out when all is well */
  309. if (sizeof(struct buffer_page) > sizeof(struct page))
  310. ring_buffer_page_too_big();
  311. /* keep it in its own cache line */
  312. buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),
  313. GFP_KERNEL);
  314. if (!buffer)
  315. return NULL;
  316. buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
  317. buffer->flags = flags;
  318. /* need at least two pages */
  319. if (buffer->pages == 1)
  320. buffer->pages++;
  321. buffer->cpumask = cpu_possible_map;
  322. buffer->cpus = nr_cpu_ids;
  323. bsize = sizeof(void *) * nr_cpu_ids;
  324. buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),
  325. GFP_KERNEL);
  326. if (!buffer->buffers)
  327. goto fail_free_buffer;
  328. for_each_buffer_cpu(buffer, cpu) {
  329. buffer->buffers[cpu] =
  330. rb_allocate_cpu_buffer(buffer, cpu);
  331. if (!buffer->buffers[cpu])
  332. goto fail_free_buffers;
  333. }
  334. mutex_init(&buffer->mutex);
  335. return buffer;
  336. fail_free_buffers:
  337. for_each_buffer_cpu(buffer, cpu) {
  338. if (buffer->buffers[cpu])
  339. rb_free_cpu_buffer(buffer->buffers[cpu]);
  340. }
  341. kfree(buffer->buffers);
  342. fail_free_buffer:
  343. kfree(buffer);
  344. return NULL;
  345. }
  346. /**
  347. * ring_buffer_free - free a ring buffer.
  348. * @buffer: the buffer to free.
  349. */
  350. void
  351. ring_buffer_free(struct ring_buffer *buffer)
  352. {
  353. int cpu;
  354. for_each_buffer_cpu(buffer, cpu)
  355. rb_free_cpu_buffer(buffer->buffers[cpu]);
  356. kfree(buffer);
  357. }
  358. static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);
  359. static void
  360. rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages)
  361. {
  362. struct buffer_page *page;
  363. struct list_head *p;
  364. unsigned i;
  365. atomic_inc(&cpu_buffer->record_disabled);
  366. synchronize_sched();
  367. for (i = 0; i < nr_pages; i++) {
  368. BUG_ON(list_empty(&cpu_buffer->pages));
  369. p = cpu_buffer->pages.next;
  370. page = list_entry(p, struct buffer_page, list);
  371. list_del_init(&page->list);
  372. free_buffer_page(page);
  373. }
  374. BUG_ON(list_empty(&cpu_buffer->pages));
  375. rb_reset_cpu(cpu_buffer);
  376. rb_check_pages(cpu_buffer);
  377. atomic_dec(&cpu_buffer->record_disabled);
  378. }
  379. static void
  380. rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer,
  381. struct list_head *pages, unsigned nr_pages)
  382. {
  383. struct buffer_page *page;
  384. struct list_head *p;
  385. unsigned i;
  386. atomic_inc(&cpu_buffer->record_disabled);
  387. synchronize_sched();
  388. for (i = 0; i < nr_pages; i++) {
  389. BUG_ON(list_empty(pages));
  390. p = pages->next;
  391. page = list_entry(p, struct buffer_page, list);
  392. list_del_init(&page->list);
  393. list_add_tail(&page->list, &cpu_buffer->pages);
  394. }
  395. rb_reset_cpu(cpu_buffer);
  396. rb_check_pages(cpu_buffer);
  397. atomic_dec(&cpu_buffer->record_disabled);
  398. }
  399. /**
  400. * ring_buffer_resize - resize the ring buffer
  401. * @buffer: the buffer to resize.
  402. * @size: the new size.
  403. *
  404. * The tracer is responsible for making sure that the buffer is
  405. * not being used while changing the size.
  406. * Note: We may be able to change the above requirement by using
  407. * RCU synchronizations.
  408. *
  409. * Minimum size is 2 * BUF_PAGE_SIZE.
  410. *
  411. * Returns -1 on failure.
  412. */
  413. int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
  414. {
  415. struct ring_buffer_per_cpu *cpu_buffer;
  416. unsigned nr_pages, rm_pages, new_pages;
  417. struct buffer_page *page, *tmp;
  418. unsigned long buffer_size;
  419. unsigned long addr;
  420. LIST_HEAD(pages);
  421. int i, cpu;
  422. size = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
  423. size *= BUF_PAGE_SIZE;
  424. buffer_size = buffer->pages * BUF_PAGE_SIZE;
  425. /* we need a minimum of two pages */
  426. if (size < BUF_PAGE_SIZE * 2)
  427. size = BUF_PAGE_SIZE * 2;
  428. if (size == buffer_size)
  429. return size;
  430. mutex_lock(&buffer->mutex);
  431. nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
  432. if (size < buffer_size) {
  433. /* easy case, just free pages */
  434. BUG_ON(nr_pages >= buffer->pages);
  435. rm_pages = buffer->pages - nr_pages;
  436. for_each_buffer_cpu(buffer, cpu) {
  437. cpu_buffer = buffer->buffers[cpu];
  438. rb_remove_pages(cpu_buffer, rm_pages);
  439. }
  440. goto out;
  441. }
  442. /*
  443. * This is a bit more difficult. We only want to add pages
  444. * when we can allocate enough for all CPUs. We do this
  445. * by allocating all the pages and storing them on a local
  446. * link list. If we succeed in our allocation, then we
  447. * add these pages to the cpu_buffers. Otherwise we just free
  448. * them all and return -ENOMEM;
  449. */
  450. BUG_ON(nr_pages <= buffer->pages);
  451. new_pages = nr_pages - buffer->pages;
  452. for_each_buffer_cpu(buffer, cpu) {
  453. for (i = 0; i < new_pages; i++) {
  454. page = kzalloc_node(ALIGN(sizeof(*page),
  455. cache_line_size()),
  456. GFP_KERNEL, cpu_to_node(cpu));
  457. if (!page)
  458. goto free_pages;
  459. list_add(&page->list, &pages);
  460. addr = __get_free_page(GFP_KERNEL);
  461. if (!addr)
  462. goto free_pages;
  463. page->page = (void *)addr;
  464. }
  465. }
  466. for_each_buffer_cpu(buffer, cpu) {
  467. cpu_buffer = buffer->buffers[cpu];
  468. rb_insert_pages(cpu_buffer, &pages, new_pages);
  469. }
  470. BUG_ON(!list_empty(&pages));
  471. out:
  472. buffer->pages = nr_pages;
  473. mutex_unlock(&buffer->mutex);
  474. return size;
  475. free_pages:
  476. list_for_each_entry_safe(page, tmp, &pages, list) {
  477. list_del_init(&page->list);
  478. free_buffer_page(page);
  479. }
  480. return -ENOMEM;
  481. }
  482. static inline int rb_null_event(struct ring_buffer_event *event)
  483. {
  484. return event->type == RINGBUF_TYPE_PADDING;
  485. }
  486. static inline void *__rb_page_index(struct buffer_page *page, unsigned index)
  487. {
  488. return page->page + index;
  489. }
  490. static inline struct ring_buffer_event *
  491. rb_reader_event(struct ring_buffer_per_cpu *cpu_buffer)
  492. {
  493. return __rb_page_index(cpu_buffer->reader_page,
  494. cpu_buffer->reader_page->read);
  495. }
  496. static inline struct ring_buffer_event *
  497. rb_head_event(struct ring_buffer_per_cpu *cpu_buffer)
  498. {
  499. return __rb_page_index(cpu_buffer->head_page,
  500. cpu_buffer->head_page->read);
  501. }
  502. static inline struct ring_buffer_event *
  503. rb_iter_head_event(struct ring_buffer_iter *iter)
  504. {
  505. return __rb_page_index(iter->head_page, iter->head);
  506. }
  507. static inline unsigned rb_page_write(struct buffer_page *bpage)
  508. {
  509. return local_read(&bpage->write);
  510. }
  511. static inline unsigned rb_page_commit(struct buffer_page *bpage)
  512. {
  513. return local_read(&bpage->commit);
  514. }
  515. /* Size is determined by what has been commited */
  516. static inline unsigned rb_page_size(struct buffer_page *bpage)
  517. {
  518. return rb_page_commit(bpage);
  519. }
  520. static inline unsigned
  521. rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer)
  522. {
  523. return rb_page_commit(cpu_buffer->commit_page);
  524. }
  525. static inline unsigned rb_head_size(struct ring_buffer_per_cpu *cpu_buffer)
  526. {
  527. return rb_page_commit(cpu_buffer->head_page);
  528. }
  529. /*
  530. * When the tail hits the head and the buffer is in overwrite mode,
  531. * the head jumps to the next page and all content on the previous
  532. * page is discarded. But before doing so, we update the overrun
  533. * variable of the buffer.
  534. */
  535. static void rb_update_overflow(struct ring_buffer_per_cpu *cpu_buffer)
  536. {
  537. struct ring_buffer_event *event;
  538. unsigned long head;
  539. for (head = 0; head < rb_head_size(cpu_buffer);
  540. head += rb_event_length(event)) {
  541. event = __rb_page_index(cpu_buffer->head_page, head);
  542. BUG_ON(rb_null_event(event));
  543. /* Only count data entries */
  544. if (event->type != RINGBUF_TYPE_DATA)
  545. continue;
  546. cpu_buffer->overrun++;
  547. cpu_buffer->entries--;
  548. }
  549. }
  550. static inline void rb_inc_page(struct ring_buffer_per_cpu *cpu_buffer,
  551. struct buffer_page **page)
  552. {
  553. struct list_head *p = (*page)->list.next;
  554. if (p == &cpu_buffer->pages)
  555. p = p->next;
  556. *page = list_entry(p, struct buffer_page, list);
  557. }
  558. static inline unsigned
  559. rb_event_index(struct ring_buffer_event *event)
  560. {
  561. unsigned long addr = (unsigned long)event;
  562. return (addr & ~PAGE_MASK) - (PAGE_SIZE - BUF_PAGE_SIZE);
  563. }
  564. static inline int
  565. rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
  566. struct ring_buffer_event *event)
  567. {
  568. unsigned long addr = (unsigned long)event;
  569. unsigned long index;
  570. index = rb_event_index(event);
  571. addr &= PAGE_MASK;
  572. return cpu_buffer->commit_page->page == (void *)addr &&
  573. rb_commit_index(cpu_buffer) == index;
  574. }
  575. static inline void
  576. rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer,
  577. struct ring_buffer_event *event)
  578. {
  579. unsigned long addr = (unsigned long)event;
  580. unsigned long index;
  581. index = rb_event_index(event);
  582. addr &= PAGE_MASK;
  583. while (cpu_buffer->commit_page->page != (void *)addr) {
  584. RB_WARN_ON(cpu_buffer,
  585. cpu_buffer->commit_page == cpu_buffer->tail_page);
  586. cpu_buffer->commit_page->commit =
  587. cpu_buffer->commit_page->write;
  588. rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
  589. cpu_buffer->write_stamp = cpu_buffer->commit_page->time_stamp;
  590. }
  591. /* Now set the commit to the event's index */
  592. local_set(&cpu_buffer->commit_page->commit, index);
  593. }
  594. static inline void
  595. rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
  596. {
  597. /*
  598. * We only race with interrupts and NMIs on this CPU.
  599. * If we own the commit event, then we can commit
  600. * all others that interrupted us, since the interruptions
  601. * are in stack format (they finish before they come
  602. * back to us). This allows us to do a simple loop to
  603. * assign the commit to the tail.
  604. */
  605. while (cpu_buffer->commit_page != cpu_buffer->tail_page) {
  606. cpu_buffer->commit_page->commit =
  607. cpu_buffer->commit_page->write;
  608. rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
  609. cpu_buffer->write_stamp = cpu_buffer->commit_page->time_stamp;
  610. /* add barrier to keep gcc from optimizing too much */
  611. barrier();
  612. }
  613. while (rb_commit_index(cpu_buffer) !=
  614. rb_page_write(cpu_buffer->commit_page)) {
  615. cpu_buffer->commit_page->commit =
  616. cpu_buffer->commit_page->write;
  617. barrier();
  618. }
  619. }
  620. static void rb_reset_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
  621. {
  622. cpu_buffer->read_stamp = cpu_buffer->reader_page->time_stamp;
  623. cpu_buffer->reader_page->read = 0;
  624. }
  625. static inline void rb_inc_iter(struct ring_buffer_iter *iter)
  626. {
  627. struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
  628. /*
  629. * The iterator could be on the reader page (it starts there).
  630. * But the head could have moved, since the reader was
  631. * found. Check for this case and assign the iterator
  632. * to the head page instead of next.
  633. */
  634. if (iter->head_page == cpu_buffer->reader_page)
  635. iter->head_page = cpu_buffer->head_page;
  636. else
  637. rb_inc_page(cpu_buffer, &iter->head_page);
  638. iter->read_stamp = iter->head_page->time_stamp;
  639. iter->head = 0;
  640. }
  641. /**
  642. * ring_buffer_update_event - update event type and data
  643. * @event: the even to update
  644. * @type: the type of event
  645. * @length: the size of the event field in the ring buffer
  646. *
  647. * Update the type and data fields of the event. The length
  648. * is the actual size that is written to the ring buffer,
  649. * and with this, we can determine what to place into the
  650. * data field.
  651. */
  652. static inline void
  653. rb_update_event(struct ring_buffer_event *event,
  654. unsigned type, unsigned length)
  655. {
  656. event->type = type;
  657. switch (type) {
  658. case RINGBUF_TYPE_PADDING:
  659. break;
  660. case RINGBUF_TYPE_TIME_EXTEND:
  661. event->len =
  662. (RB_LEN_TIME_EXTEND + (RB_ALIGNMENT-1))
  663. >> RB_ALIGNMENT_SHIFT;
  664. break;
  665. case RINGBUF_TYPE_TIME_STAMP:
  666. event->len =
  667. (RB_LEN_TIME_STAMP + (RB_ALIGNMENT-1))
  668. >> RB_ALIGNMENT_SHIFT;
  669. break;
  670. case RINGBUF_TYPE_DATA:
  671. length -= RB_EVNT_HDR_SIZE;
  672. if (length > RB_MAX_SMALL_DATA) {
  673. event->len = 0;
  674. event->array[0] = length;
  675. } else
  676. event->len =
  677. (length + (RB_ALIGNMENT-1))
  678. >> RB_ALIGNMENT_SHIFT;
  679. break;
  680. default:
  681. BUG();
  682. }
  683. }
  684. static inline unsigned rb_calculate_event_length(unsigned length)
  685. {
  686. struct ring_buffer_event event; /* Used only for sizeof array */
  687. /* zero length can cause confusions */
  688. if (!length)
  689. length = 1;
  690. if (length > RB_MAX_SMALL_DATA)
  691. length += sizeof(event.array[0]);
  692. length += RB_EVNT_HDR_SIZE;
  693. length = ALIGN(length, RB_ALIGNMENT);
  694. return length;
  695. }
  696. static struct ring_buffer_event *
  697. __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
  698. unsigned type, unsigned long length, u64 *ts)
  699. {
  700. struct buffer_page *tail_page, *head_page, *reader_page;
  701. unsigned long tail, write;
  702. struct ring_buffer *buffer = cpu_buffer->buffer;
  703. struct ring_buffer_event *event;
  704. unsigned long flags;
  705. tail_page = cpu_buffer->tail_page;
  706. write = local_add_return(length, &tail_page->write);
  707. tail = write - length;
  708. /* See if we shot pass the end of this buffer page */
  709. if (write > BUF_PAGE_SIZE) {
  710. struct buffer_page *next_page = tail_page;
  711. spin_lock_irqsave(&cpu_buffer->lock, flags);
  712. rb_inc_page(cpu_buffer, &next_page);
  713. head_page = cpu_buffer->head_page;
  714. reader_page = cpu_buffer->reader_page;
  715. /* we grabbed the lock before incrementing */
  716. RB_WARN_ON(cpu_buffer, next_page == reader_page);
  717. /*
  718. * If for some reason, we had an interrupt storm that made
  719. * it all the way around the buffer, bail, and warn
  720. * about it.
  721. */
  722. if (unlikely(next_page == cpu_buffer->commit_page)) {
  723. WARN_ON_ONCE(1);
  724. goto out_unlock;
  725. }
  726. if (next_page == head_page) {
  727. if (!(buffer->flags & RB_FL_OVERWRITE)) {
  728. /* reset write */
  729. if (tail <= BUF_PAGE_SIZE)
  730. local_set(&tail_page->write, tail);
  731. goto out_unlock;
  732. }
  733. /* tail_page has not moved yet? */
  734. if (tail_page == cpu_buffer->tail_page) {
  735. /* count overflows */
  736. rb_update_overflow(cpu_buffer);
  737. rb_inc_page(cpu_buffer, &head_page);
  738. cpu_buffer->head_page = head_page;
  739. cpu_buffer->head_page->read = 0;
  740. }
  741. }
  742. /*
  743. * If the tail page is still the same as what we think
  744. * it is, then it is up to us to update the tail
  745. * pointer.
  746. */
  747. if (tail_page == cpu_buffer->tail_page) {
  748. local_set(&next_page->write, 0);
  749. local_set(&next_page->commit, 0);
  750. cpu_buffer->tail_page = next_page;
  751. /* reread the time stamp */
  752. *ts = ring_buffer_time_stamp(cpu_buffer->cpu);
  753. cpu_buffer->tail_page->time_stamp = *ts;
  754. }
  755. /*
  756. * The actual tail page has moved forward.
  757. */
  758. if (tail < BUF_PAGE_SIZE) {
  759. /* Mark the rest of the page with padding */
  760. event = __rb_page_index(tail_page, tail);
  761. event->type = RINGBUF_TYPE_PADDING;
  762. }
  763. if (tail <= BUF_PAGE_SIZE)
  764. /* Set the write back to the previous setting */
  765. local_set(&tail_page->write, tail);
  766. /*
  767. * If this was a commit entry that failed,
  768. * increment that too
  769. */
  770. if (tail_page == cpu_buffer->commit_page &&
  771. tail == rb_commit_index(cpu_buffer)) {
  772. rb_set_commit_to_write(cpu_buffer);
  773. }
  774. spin_unlock_irqrestore(&cpu_buffer->lock, flags);
  775. /* fail and let the caller try again */
  776. return ERR_PTR(-EAGAIN);
  777. }
  778. /* We reserved something on the buffer */
  779. BUG_ON(write > BUF_PAGE_SIZE);
  780. event = __rb_page_index(tail_page, tail);
  781. rb_update_event(event, type, length);
  782. /*
  783. * If this is a commit and the tail is zero, then update
  784. * this page's time stamp.
  785. */
  786. if (!tail && rb_is_commit(cpu_buffer, event))
  787. cpu_buffer->commit_page->time_stamp = *ts;
  788. return event;
  789. out_unlock:
  790. spin_unlock_irqrestore(&cpu_buffer->lock, flags);
  791. return NULL;
  792. }
  793. static int
  794. rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
  795. u64 *ts, u64 *delta)
  796. {
  797. struct ring_buffer_event *event;
  798. static int once;
  799. int ret;
  800. if (unlikely(*delta > (1ULL << 59) && !once++)) {
  801. printk(KERN_WARNING "Delta way too big! %llu"
  802. " ts=%llu write stamp = %llu\n",
  803. (unsigned long long)*delta,
  804. (unsigned long long)*ts,
  805. (unsigned long long)cpu_buffer->write_stamp);
  806. WARN_ON(1);
  807. }
  808. /*
  809. * The delta is too big, we to add a
  810. * new timestamp.
  811. */
  812. event = __rb_reserve_next(cpu_buffer,
  813. RINGBUF_TYPE_TIME_EXTEND,
  814. RB_LEN_TIME_EXTEND,
  815. ts);
  816. if (!event)
  817. return -EBUSY;
  818. if (PTR_ERR(event) == -EAGAIN)
  819. return -EAGAIN;
  820. /* Only a commited time event can update the write stamp */
  821. if (rb_is_commit(cpu_buffer, event)) {
  822. /*
  823. * If this is the first on the page, then we need to
  824. * update the page itself, and just put in a zero.
  825. */
  826. if (rb_event_index(event)) {
  827. event->time_delta = *delta & TS_MASK;
  828. event->array[0] = *delta >> TS_SHIFT;
  829. } else {
  830. cpu_buffer->commit_page->time_stamp = *ts;
  831. event->time_delta = 0;
  832. event->array[0] = 0;
  833. }
  834. cpu_buffer->write_stamp = *ts;
  835. /* let the caller know this was the commit */
  836. ret = 1;
  837. } else {
  838. /* Darn, this is just wasted space */
  839. event->time_delta = 0;
  840. event->array[0] = 0;
  841. ret = 0;
  842. }
  843. *delta = 0;
  844. return ret;
  845. }
  846. static struct ring_buffer_event *
  847. rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer,
  848. unsigned type, unsigned long length)
  849. {
  850. struct ring_buffer_event *event;
  851. u64 ts, delta;
  852. int commit = 0;
  853. again:
  854. ts = ring_buffer_time_stamp(cpu_buffer->cpu);
  855. /*
  856. * Only the first commit can update the timestamp.
  857. * Yes there is a race here. If an interrupt comes in
  858. * just after the conditional and it traces too, then it
  859. * will also check the deltas. More than one timestamp may
  860. * also be made. But only the entry that did the actual
  861. * commit will be something other than zero.
  862. */
  863. if (cpu_buffer->tail_page == cpu_buffer->commit_page &&
  864. rb_page_write(cpu_buffer->tail_page) ==
  865. rb_commit_index(cpu_buffer)) {
  866. delta = ts - cpu_buffer->write_stamp;
  867. /* make sure this delta is calculated here */
  868. barrier();
  869. /* Did the write stamp get updated already? */
  870. if (unlikely(ts < cpu_buffer->write_stamp))
  871. goto again;
  872. if (test_time_stamp(delta)) {
  873. commit = rb_add_time_stamp(cpu_buffer, &ts, &delta);
  874. if (commit == -EBUSY)
  875. return NULL;
  876. if (commit == -EAGAIN)
  877. goto again;
  878. RB_WARN_ON(cpu_buffer, commit < 0);
  879. }
  880. } else
  881. /* Non commits have zero deltas */
  882. delta = 0;
  883. event = __rb_reserve_next(cpu_buffer, type, length, &ts);
  884. if (PTR_ERR(event) == -EAGAIN)
  885. goto again;
  886. if (!event) {
  887. if (unlikely(commit))
  888. /*
  889. * Ouch! We needed a timestamp and it was commited. But
  890. * we didn't get our event reserved.
  891. */
  892. rb_set_commit_to_write(cpu_buffer);
  893. return NULL;
  894. }
  895. /*
  896. * If the timestamp was commited, make the commit our entry
  897. * now so that we will update it when needed.
  898. */
  899. if (commit)
  900. rb_set_commit_event(cpu_buffer, event);
  901. else if (!rb_is_commit(cpu_buffer, event))
  902. delta = 0;
  903. event->time_delta = delta;
  904. return event;
  905. }
  906. static DEFINE_PER_CPU(int, rb_need_resched);
  907. /**
  908. * ring_buffer_lock_reserve - reserve a part of the buffer
  909. * @buffer: the ring buffer to reserve from
  910. * @length: the length of the data to reserve (excluding event header)
  911. * @flags: a pointer to save the interrupt flags
  912. *
  913. * Returns a reseverd event on the ring buffer to copy directly to.
  914. * The user of this interface will need to get the body to write into
  915. * and can use the ring_buffer_event_data() interface.
  916. *
  917. * The length is the length of the data needed, not the event length
  918. * which also includes the event header.
  919. *
  920. * Must be paired with ring_buffer_unlock_commit, unless NULL is returned.
  921. * If NULL is returned, then nothing has been allocated or locked.
  922. */
  923. struct ring_buffer_event *
  924. ring_buffer_lock_reserve(struct ring_buffer *buffer,
  925. unsigned long length,
  926. unsigned long *flags)
  927. {
  928. struct ring_buffer_per_cpu *cpu_buffer;
  929. struct ring_buffer_event *event;
  930. int cpu, resched;
  931. if (atomic_read(&buffer->record_disabled))
  932. return NULL;
  933. /* If we are tracing schedule, we don't want to recurse */
  934. resched = need_resched();
  935. preempt_disable_notrace();
  936. cpu = raw_smp_processor_id();
  937. if (!cpu_isset(cpu, buffer->cpumask))
  938. goto out;
  939. cpu_buffer = buffer->buffers[cpu];
  940. if (atomic_read(&cpu_buffer->record_disabled))
  941. goto out;
  942. length = rb_calculate_event_length(length);
  943. if (length > BUF_PAGE_SIZE)
  944. goto out;
  945. event = rb_reserve_next_event(cpu_buffer, RINGBUF_TYPE_DATA, length);
  946. if (!event)
  947. goto out;
  948. /*
  949. * Need to store resched state on this cpu.
  950. * Only the first needs to.
  951. */
  952. if (preempt_count() == 1)
  953. per_cpu(rb_need_resched, cpu) = resched;
  954. return event;
  955. out:
  956. if (resched)
  957. preempt_enable_notrace();
  958. else
  959. preempt_enable_notrace();
  960. return NULL;
  961. }
  962. static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
  963. struct ring_buffer_event *event)
  964. {
  965. cpu_buffer->entries++;
  966. /* Only process further if we own the commit */
  967. if (!rb_is_commit(cpu_buffer, event))
  968. return;
  969. cpu_buffer->write_stamp += event->time_delta;
  970. rb_set_commit_to_write(cpu_buffer);
  971. }
  972. /**
  973. * ring_buffer_unlock_commit - commit a reserved
  974. * @buffer: The buffer to commit to
  975. * @event: The event pointer to commit.
  976. * @flags: the interrupt flags received from ring_buffer_lock_reserve.
  977. *
  978. * This commits the data to the ring buffer, and releases any locks held.
  979. *
  980. * Must be paired with ring_buffer_lock_reserve.
  981. */
  982. int ring_buffer_unlock_commit(struct ring_buffer *buffer,
  983. struct ring_buffer_event *event,
  984. unsigned long flags)
  985. {
  986. struct ring_buffer_per_cpu *cpu_buffer;
  987. int cpu = raw_smp_processor_id();
  988. cpu_buffer = buffer->buffers[cpu];
  989. rb_commit(cpu_buffer, event);
  990. /*
  991. * Only the last preempt count needs to restore preemption.
  992. */
  993. if (preempt_count() == 1) {
  994. if (per_cpu(rb_need_resched, cpu))
  995. preempt_enable_no_resched_notrace();
  996. else
  997. preempt_enable_notrace();
  998. } else
  999. preempt_enable_no_resched_notrace();
  1000. return 0;
  1001. }
  1002. /**
  1003. * ring_buffer_write - write data to the buffer without reserving
  1004. * @buffer: The ring buffer to write to.
  1005. * @length: The length of the data being written (excluding the event header)
  1006. * @data: The data to write to the buffer.
  1007. *
  1008. * This is like ring_buffer_lock_reserve and ring_buffer_unlock_commit as
  1009. * one function. If you already have the data to write to the buffer, it
  1010. * may be easier to simply call this function.
  1011. *
  1012. * Note, like ring_buffer_lock_reserve, the length is the length of the data
  1013. * and not the length of the event which would hold the header.
  1014. */
  1015. int ring_buffer_write(struct ring_buffer *buffer,
  1016. unsigned long length,
  1017. void *data)
  1018. {
  1019. struct ring_buffer_per_cpu *cpu_buffer;
  1020. struct ring_buffer_event *event;
  1021. unsigned long event_length;
  1022. void *body;
  1023. int ret = -EBUSY;
  1024. int cpu, resched;
  1025. if (atomic_read(&buffer->record_disabled))
  1026. return -EBUSY;
  1027. resched = need_resched();
  1028. preempt_disable_notrace();
  1029. cpu = raw_smp_processor_id();
  1030. if (!cpu_isset(cpu, buffer->cpumask))
  1031. goto out;
  1032. cpu_buffer = buffer->buffers[cpu];
  1033. if (atomic_read(&cpu_buffer->record_disabled))
  1034. goto out;
  1035. event_length = rb_calculate_event_length(length);
  1036. event = rb_reserve_next_event(cpu_buffer,
  1037. RINGBUF_TYPE_DATA, event_length);
  1038. if (!event)
  1039. goto out;
  1040. body = rb_event_data(event);
  1041. memcpy(body, data, length);
  1042. rb_commit(cpu_buffer, event);
  1043. ret = 0;
  1044. out:
  1045. if (resched)
  1046. preempt_enable_no_resched_notrace();
  1047. else
  1048. preempt_enable_notrace();
  1049. return ret;
  1050. }
  1051. static inline int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
  1052. {
  1053. struct buffer_page *reader = cpu_buffer->reader_page;
  1054. struct buffer_page *head = cpu_buffer->head_page;
  1055. struct buffer_page *commit = cpu_buffer->commit_page;
  1056. return reader->read == rb_page_commit(reader) &&
  1057. (commit == reader ||
  1058. (commit == head &&
  1059. head->read == rb_page_commit(commit)));
  1060. }
  1061. /**
  1062. * ring_buffer_record_disable - stop all writes into the buffer
  1063. * @buffer: The ring buffer to stop writes to.
  1064. *
  1065. * This prevents all writes to the buffer. Any attempt to write
  1066. * to the buffer after this will fail and return NULL.
  1067. *
  1068. * The caller should call synchronize_sched() after this.
  1069. */
  1070. void ring_buffer_record_disable(struct ring_buffer *buffer)
  1071. {
  1072. atomic_inc(&buffer->record_disabled);
  1073. }
  1074. /**
  1075. * ring_buffer_record_enable - enable writes to the buffer
  1076. * @buffer: The ring buffer to enable writes
  1077. *
  1078. * Note, multiple disables will need the same number of enables
  1079. * to truely enable the writing (much like preempt_disable).
  1080. */
  1081. void ring_buffer_record_enable(struct ring_buffer *buffer)
  1082. {
  1083. atomic_dec(&buffer->record_disabled);
  1084. }
  1085. /**
  1086. * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer
  1087. * @buffer: The ring buffer to stop writes to.
  1088. * @cpu: The CPU buffer to stop
  1089. *
  1090. * This prevents all writes to the buffer. Any attempt to write
  1091. * to the buffer after this will fail and return NULL.
  1092. *
  1093. * The caller should call synchronize_sched() after this.
  1094. */
  1095. void ring_buffer_record_disable_cpu(struct ring_buffer *buffer, int cpu)
  1096. {
  1097. struct ring_buffer_per_cpu *cpu_buffer;
  1098. if (!cpu_isset(cpu, buffer->cpumask))
  1099. return;
  1100. cpu_buffer = buffer->buffers[cpu];
  1101. atomic_inc(&cpu_buffer->record_disabled);
  1102. }
  1103. /**
  1104. * ring_buffer_record_enable_cpu - enable writes to the buffer
  1105. * @buffer: The ring buffer to enable writes
  1106. * @cpu: The CPU to enable.
  1107. *
  1108. * Note, multiple disables will need the same number of enables
  1109. * to truely enable the writing (much like preempt_disable).
  1110. */
  1111. void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu)
  1112. {
  1113. struct ring_buffer_per_cpu *cpu_buffer;
  1114. if (!cpu_isset(cpu, buffer->cpumask))
  1115. return;
  1116. cpu_buffer = buffer->buffers[cpu];
  1117. atomic_dec(&cpu_buffer->record_disabled);
  1118. }
  1119. /**
  1120. * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
  1121. * @buffer: The ring buffer
  1122. * @cpu: The per CPU buffer to get the entries from.
  1123. */
  1124. unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
  1125. {
  1126. struct ring_buffer_per_cpu *cpu_buffer;
  1127. if (!cpu_isset(cpu, buffer->cpumask))
  1128. return 0;
  1129. cpu_buffer = buffer->buffers[cpu];
  1130. return cpu_buffer->entries;
  1131. }
  1132. /**
  1133. * ring_buffer_overrun_cpu - get the number of overruns in a cpu_buffer
  1134. * @buffer: The ring buffer
  1135. * @cpu: The per CPU buffer to get the number of overruns from
  1136. */
  1137. unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
  1138. {
  1139. struct ring_buffer_per_cpu *cpu_buffer;
  1140. if (!cpu_isset(cpu, buffer->cpumask))
  1141. return 0;
  1142. cpu_buffer = buffer->buffers[cpu];
  1143. return cpu_buffer->overrun;
  1144. }
  1145. /**
  1146. * ring_buffer_entries - get the number of entries in a buffer
  1147. * @buffer: The ring buffer
  1148. *
  1149. * Returns the total number of entries in the ring buffer
  1150. * (all CPU entries)
  1151. */
  1152. unsigned long ring_buffer_entries(struct ring_buffer *buffer)
  1153. {
  1154. struct ring_buffer_per_cpu *cpu_buffer;
  1155. unsigned long entries = 0;
  1156. int cpu;
  1157. /* if you care about this being correct, lock the buffer */
  1158. for_each_buffer_cpu(buffer, cpu) {
  1159. cpu_buffer = buffer->buffers[cpu];
  1160. entries += cpu_buffer->entries;
  1161. }
  1162. return entries;
  1163. }
  1164. /**
  1165. * ring_buffer_overrun_cpu - get the number of overruns in buffer
  1166. * @buffer: The ring buffer
  1167. *
  1168. * Returns the total number of overruns in the ring buffer
  1169. * (all CPU entries)
  1170. */
  1171. unsigned long ring_buffer_overruns(struct ring_buffer *buffer)
  1172. {
  1173. struct ring_buffer_per_cpu *cpu_buffer;
  1174. unsigned long overruns = 0;
  1175. int cpu;
  1176. /* if you care about this being correct, lock the buffer */
  1177. for_each_buffer_cpu(buffer, cpu) {
  1178. cpu_buffer = buffer->buffers[cpu];
  1179. overruns += cpu_buffer->overrun;
  1180. }
  1181. return overruns;
  1182. }
  1183. /**
  1184. * ring_buffer_iter_reset - reset an iterator
  1185. * @iter: The iterator to reset
  1186. *
  1187. * Resets the iterator, so that it will start from the beginning
  1188. * again.
  1189. */
  1190. void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
  1191. {
  1192. struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
  1193. /* Iterator usage is expected to have record disabled */
  1194. if (list_empty(&cpu_buffer->reader_page->list)) {
  1195. iter->head_page = cpu_buffer->head_page;
  1196. iter->head = cpu_buffer->head_page->read;
  1197. } else {
  1198. iter->head_page = cpu_buffer->reader_page;
  1199. iter->head = cpu_buffer->reader_page->read;
  1200. }
  1201. if (iter->head)
  1202. iter->read_stamp = cpu_buffer->read_stamp;
  1203. else
  1204. iter->read_stamp = iter->head_page->time_stamp;
  1205. }
  1206. /**
  1207. * ring_buffer_iter_empty - check if an iterator has no more to read
  1208. * @iter: The iterator to check
  1209. */
  1210. int ring_buffer_iter_empty(struct ring_buffer_iter *iter)
  1211. {
  1212. struct ring_buffer_per_cpu *cpu_buffer;
  1213. cpu_buffer = iter->cpu_buffer;
  1214. return iter->head_page == cpu_buffer->commit_page &&
  1215. iter->head == rb_commit_index(cpu_buffer);
  1216. }
  1217. static void
  1218. rb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer,
  1219. struct ring_buffer_event *event)
  1220. {
  1221. u64 delta;
  1222. switch (event->type) {
  1223. case RINGBUF_TYPE_PADDING:
  1224. return;
  1225. case RINGBUF_TYPE_TIME_EXTEND:
  1226. delta = event->array[0];
  1227. delta <<= TS_SHIFT;
  1228. delta += event->time_delta;
  1229. cpu_buffer->read_stamp += delta;
  1230. return;
  1231. case RINGBUF_TYPE_TIME_STAMP:
  1232. /* FIXME: not implemented */
  1233. return;
  1234. case RINGBUF_TYPE_DATA:
  1235. cpu_buffer->read_stamp += event->time_delta;
  1236. return;
  1237. default:
  1238. BUG();
  1239. }
  1240. return;
  1241. }
  1242. static void
  1243. rb_update_iter_read_stamp(struct ring_buffer_iter *iter,
  1244. struct ring_buffer_event *event)
  1245. {
  1246. u64 delta;
  1247. switch (event->type) {
  1248. case RINGBUF_TYPE_PADDING:
  1249. return;
  1250. case RINGBUF_TYPE_TIME_EXTEND:
  1251. delta = event->array[0];
  1252. delta <<= TS_SHIFT;
  1253. delta += event->time_delta;
  1254. iter->read_stamp += delta;
  1255. return;
  1256. case RINGBUF_TYPE_TIME_STAMP:
  1257. /* FIXME: not implemented */
  1258. return;
  1259. case RINGBUF_TYPE_DATA:
  1260. iter->read_stamp += event->time_delta;
  1261. return;
  1262. default:
  1263. BUG();
  1264. }
  1265. return;
  1266. }
  1267. static struct buffer_page *
  1268. rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
  1269. {
  1270. struct buffer_page *reader = NULL;
  1271. unsigned long flags;
  1272. spin_lock_irqsave(&cpu_buffer->lock, flags);
  1273. again:
  1274. reader = cpu_buffer->reader_page;
  1275. /* If there's more to read, return this page */
  1276. if (cpu_buffer->reader_page->read < rb_page_size(reader))
  1277. goto out;
  1278. /* Never should we have an index greater than the size */
  1279. RB_WARN_ON(cpu_buffer,
  1280. cpu_buffer->reader_page->read > rb_page_size(reader));
  1281. /* check if we caught up to the tail */
  1282. reader = NULL;
  1283. if (cpu_buffer->commit_page == cpu_buffer->reader_page)
  1284. goto out;
  1285. /*
  1286. * Splice the empty reader page into the list around the head.
  1287. * Reset the reader page to size zero.
  1288. */
  1289. reader = cpu_buffer->head_page;
  1290. cpu_buffer->reader_page->list.next = reader->list.next;
  1291. cpu_buffer->reader_page->list.prev = reader->list.prev;
  1292. local_set(&cpu_buffer->reader_page->write, 0);
  1293. local_set(&cpu_buffer->reader_page->commit, 0);
  1294. /* Make the reader page now replace the head */
  1295. reader->list.prev->next = &cpu_buffer->reader_page->list;
  1296. reader->list.next->prev = &cpu_buffer->reader_page->list;
  1297. /*
  1298. * If the tail is on the reader, then we must set the head
  1299. * to the inserted page, otherwise we set it one before.
  1300. */
  1301. cpu_buffer->head_page = cpu_buffer->reader_page;
  1302. if (cpu_buffer->commit_page != reader)
  1303. rb_inc_page(cpu_buffer, &cpu_buffer->head_page);
  1304. /* Finally update the reader page to the new head */
  1305. cpu_buffer->reader_page = reader;
  1306. rb_reset_reader_page(cpu_buffer);
  1307. goto again;
  1308. out:
  1309. spin_unlock_irqrestore(&cpu_buffer->lock, flags);
  1310. return reader;
  1311. }
  1312. static void rb_advance_reader(struct ring_buffer_per_cpu *cpu_buffer)
  1313. {
  1314. struct ring_buffer_event *event;
  1315. struct buffer_page *reader;
  1316. unsigned length;
  1317. reader = rb_get_reader_page(cpu_buffer);
  1318. /* This function should not be called when buffer is empty */
  1319. BUG_ON(!reader);
  1320. event = rb_reader_event(cpu_buffer);
  1321. if (event->type == RINGBUF_TYPE_DATA)
  1322. cpu_buffer->entries--;
  1323. rb_update_read_stamp(cpu_buffer, event);
  1324. length = rb_event_length(event);
  1325. cpu_buffer->reader_page->read += length;
  1326. }
  1327. static void rb_advance_iter(struct ring_buffer_iter *iter)
  1328. {
  1329. struct ring_buffer *buffer;
  1330. struct ring_buffer_per_cpu *cpu_buffer;
  1331. struct ring_buffer_event *event;
  1332. unsigned length;
  1333. cpu_buffer = iter->cpu_buffer;
  1334. buffer = cpu_buffer->buffer;
  1335. /*
  1336. * Check if we are at the end of the buffer.
  1337. */
  1338. if (iter->head >= rb_page_size(iter->head_page)) {
  1339. BUG_ON(iter->head_page == cpu_buffer->commit_page);
  1340. rb_inc_iter(iter);
  1341. return;
  1342. }
  1343. event = rb_iter_head_event(iter);
  1344. length = rb_event_length(event);
  1345. /*
  1346. * This should not be called to advance the header if we are
  1347. * at the tail of the buffer.
  1348. */
  1349. BUG_ON((iter->head_page == cpu_buffer->commit_page) &&
  1350. (iter->head + length > rb_commit_index(cpu_buffer)));
  1351. rb_update_iter_read_stamp(iter, event);
  1352. iter->head += length;
  1353. /* check for end of page padding */
  1354. if ((iter->head >= rb_page_size(iter->head_page)) &&
  1355. (iter->head_page != cpu_buffer->commit_page))
  1356. rb_advance_iter(iter);
  1357. }
  1358. /**
  1359. * ring_buffer_peek - peek at the next event to be read
  1360. * @buffer: The ring buffer to read
  1361. * @cpu: The cpu to peak at
  1362. * @ts: The timestamp counter of this event.
  1363. *
  1364. * This will return the event that will be read next, but does
  1365. * not consume the data.
  1366. */
  1367. struct ring_buffer_event *
  1368. ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
  1369. {
  1370. struct ring_buffer_per_cpu *cpu_buffer;
  1371. struct ring_buffer_event *event;
  1372. struct buffer_page *reader;
  1373. if (!cpu_isset(cpu, buffer->cpumask))
  1374. return NULL;
  1375. cpu_buffer = buffer->buffers[cpu];
  1376. again:
  1377. reader = rb_get_reader_page(cpu_buffer);
  1378. if (!reader)
  1379. return NULL;
  1380. event = rb_reader_event(cpu_buffer);
  1381. switch (event->type) {
  1382. case RINGBUF_TYPE_PADDING:
  1383. RB_WARN_ON(cpu_buffer, 1);
  1384. rb_advance_reader(cpu_buffer);
  1385. return NULL;
  1386. case RINGBUF_TYPE_TIME_EXTEND:
  1387. /* Internal data, OK to advance */
  1388. rb_advance_reader(cpu_buffer);
  1389. goto again;
  1390. case RINGBUF_TYPE_TIME_STAMP:
  1391. /* FIXME: not implemented */
  1392. rb_advance_reader(cpu_buffer);
  1393. goto again;
  1394. case RINGBUF_TYPE_DATA:
  1395. if (ts) {
  1396. *ts = cpu_buffer->read_stamp + event->time_delta;
  1397. ring_buffer_normalize_time_stamp(cpu_buffer->cpu, ts);
  1398. }
  1399. return event;
  1400. default:
  1401. BUG();
  1402. }
  1403. return NULL;
  1404. }
  1405. /**
  1406. * ring_buffer_iter_peek - peek at the next event to be read
  1407. * @iter: The ring buffer iterator
  1408. * @ts: The timestamp counter of this event.
  1409. *
  1410. * This will return the event that will be read next, but does
  1411. * not increment the iterator.
  1412. */
  1413. struct ring_buffer_event *
  1414. ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
  1415. {
  1416. struct ring_buffer *buffer;
  1417. struct ring_buffer_per_cpu *cpu_buffer;
  1418. struct ring_buffer_event *event;
  1419. if (ring_buffer_iter_empty(iter))
  1420. return NULL;
  1421. cpu_buffer = iter->cpu_buffer;
  1422. buffer = cpu_buffer->buffer;
  1423. again:
  1424. if (rb_per_cpu_empty(cpu_buffer))
  1425. return NULL;
  1426. event = rb_iter_head_event(iter);
  1427. switch (event->type) {
  1428. case RINGBUF_TYPE_PADDING:
  1429. rb_inc_iter(iter);
  1430. goto again;
  1431. case RINGBUF_TYPE_TIME_EXTEND:
  1432. /* Internal data, OK to advance */
  1433. rb_advance_iter(iter);
  1434. goto again;
  1435. case RINGBUF_TYPE_TIME_STAMP:
  1436. /* FIXME: not implemented */
  1437. rb_advance_iter(iter);
  1438. goto again;
  1439. case RINGBUF_TYPE_DATA:
  1440. if (ts) {
  1441. *ts = iter->read_stamp + event->time_delta;
  1442. ring_buffer_normalize_time_stamp(cpu_buffer->cpu, ts);
  1443. }
  1444. return event;
  1445. default:
  1446. BUG();
  1447. }
  1448. return NULL;
  1449. }
  1450. /**
  1451. * ring_buffer_consume - return an event and consume it
  1452. * @buffer: The ring buffer to get the next event from
  1453. *
  1454. * Returns the next event in the ring buffer, and that event is consumed.
  1455. * Meaning, that sequential reads will keep returning a different event,
  1456. * and eventually empty the ring buffer if the producer is slower.
  1457. */
  1458. struct ring_buffer_event *
  1459. ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
  1460. {
  1461. struct ring_buffer_per_cpu *cpu_buffer;
  1462. struct ring_buffer_event *event;
  1463. if (!cpu_isset(cpu, buffer->cpumask))
  1464. return NULL;
  1465. event = ring_buffer_peek(buffer, cpu, ts);
  1466. if (!event)
  1467. return NULL;
  1468. cpu_buffer = buffer->buffers[cpu];
  1469. rb_advance_reader(cpu_buffer);
  1470. return event;
  1471. }
  1472. /**
  1473. * ring_buffer_read_start - start a non consuming read of the buffer
  1474. * @buffer: The ring buffer to read from
  1475. * @cpu: The cpu buffer to iterate over
  1476. *
  1477. * This starts up an iteration through the buffer. It also disables
  1478. * the recording to the buffer until the reading is finished.
  1479. * This prevents the reading from being corrupted. This is not
  1480. * a consuming read, so a producer is not expected.
  1481. *
  1482. * Must be paired with ring_buffer_finish.
  1483. */
  1484. struct ring_buffer_iter *
  1485. ring_buffer_read_start(struct ring_buffer *buffer, int cpu)
  1486. {
  1487. struct ring_buffer_per_cpu *cpu_buffer;
  1488. struct ring_buffer_iter *iter;
  1489. unsigned long flags;
  1490. if (!cpu_isset(cpu, buffer->cpumask))
  1491. return NULL;
  1492. iter = kmalloc(sizeof(*iter), GFP_KERNEL);
  1493. if (!iter)
  1494. return NULL;
  1495. cpu_buffer = buffer->buffers[cpu];
  1496. iter->cpu_buffer = cpu_buffer;
  1497. atomic_inc(&cpu_buffer->record_disabled);
  1498. synchronize_sched();
  1499. spin_lock_irqsave(&cpu_buffer->lock, flags);
  1500. ring_buffer_iter_reset(iter);
  1501. spin_unlock_irqrestore(&cpu_buffer->lock, flags);
  1502. return iter;
  1503. }
  1504. /**
  1505. * ring_buffer_finish - finish reading the iterator of the buffer
  1506. * @iter: The iterator retrieved by ring_buffer_start
  1507. *
  1508. * This re-enables the recording to the buffer, and frees the
  1509. * iterator.
  1510. */
  1511. void
  1512. ring_buffer_read_finish(struct ring_buffer_iter *iter)
  1513. {
  1514. struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
  1515. atomic_dec(&cpu_buffer->record_disabled);
  1516. kfree(iter);
  1517. }
  1518. /**
  1519. * ring_buffer_read - read the next item in the ring buffer by the iterator
  1520. * @iter: The ring buffer iterator
  1521. * @ts: The time stamp of the event read.
  1522. *
  1523. * This reads the next event in the ring buffer and increments the iterator.
  1524. */
  1525. struct ring_buffer_event *
  1526. ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
  1527. {
  1528. struct ring_buffer_event *event;
  1529. event = ring_buffer_iter_peek(iter, ts);
  1530. if (!event)
  1531. return NULL;
  1532. rb_advance_iter(iter);
  1533. return event;
  1534. }
  1535. /**
  1536. * ring_buffer_size - return the size of the ring buffer (in bytes)
  1537. * @buffer: The ring buffer.
  1538. */
  1539. unsigned long ring_buffer_size(struct ring_buffer *buffer)
  1540. {
  1541. return BUF_PAGE_SIZE * buffer->pages;
  1542. }
  1543. static void
  1544. rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
  1545. {
  1546. cpu_buffer->head_page
  1547. = list_entry(cpu_buffer->pages.next, struct buffer_page, list);
  1548. local_set(&cpu_buffer->head_page->write, 0);
  1549. local_set(&cpu_buffer->head_page->commit, 0);
  1550. cpu_buffer->head_page->read = 0;
  1551. cpu_buffer->tail_page = cpu_buffer->head_page;
  1552. cpu_buffer->commit_page = cpu_buffer->head_page;
  1553. INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
  1554. local_set(&cpu_buffer->reader_page->write, 0);
  1555. local_set(&cpu_buffer->reader_page->commit, 0);
  1556. cpu_buffer->reader_page->read = 0;
  1557. cpu_buffer->overrun = 0;
  1558. cpu_buffer->entries = 0;
  1559. }
  1560. /**
  1561. * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
  1562. * @buffer: The ring buffer to reset a per cpu buffer of
  1563. * @cpu: The CPU buffer to be reset
  1564. */
  1565. void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
  1566. {
  1567. struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
  1568. unsigned long flags;
  1569. if (!cpu_isset(cpu, buffer->cpumask))
  1570. return;
  1571. spin_lock_irqsave(&cpu_buffer->lock, flags);
  1572. rb_reset_cpu(cpu_buffer);
  1573. spin_unlock_irqrestore(&cpu_buffer->lock, flags);
  1574. }
  1575. /**
  1576. * ring_buffer_reset - reset a ring buffer
  1577. * @buffer: The ring buffer to reset all cpu buffers
  1578. */
  1579. void ring_buffer_reset(struct ring_buffer *buffer)
  1580. {
  1581. int cpu;
  1582. for_each_buffer_cpu(buffer, cpu)
  1583. ring_buffer_reset_cpu(buffer, cpu);
  1584. }
  1585. /**
  1586. * rind_buffer_empty - is the ring buffer empty?
  1587. * @buffer: The ring buffer to test
  1588. */
  1589. int ring_buffer_empty(struct ring_buffer *buffer)
  1590. {
  1591. struct ring_buffer_per_cpu *cpu_buffer;
  1592. int cpu;
  1593. /* yes this is racy, but if you don't like the race, lock the buffer */
  1594. for_each_buffer_cpu(buffer, cpu) {
  1595. cpu_buffer = buffer->buffers[cpu];
  1596. if (!rb_per_cpu_empty(cpu_buffer))
  1597. return 0;
  1598. }
  1599. return 1;
  1600. }
  1601. /**
  1602. * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
  1603. * @buffer: The ring buffer
  1604. * @cpu: The CPU buffer to test
  1605. */
  1606. int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
  1607. {
  1608. struct ring_buffer_per_cpu *cpu_buffer;
  1609. if (!cpu_isset(cpu, buffer->cpumask))
  1610. return 1;
  1611. cpu_buffer = buffer->buffers[cpu];
  1612. return rb_per_cpu_empty(cpu_buffer);
  1613. }
  1614. /**
  1615. * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
  1616. * @buffer_a: One buffer to swap with
  1617. * @buffer_b: The other buffer to swap with
  1618. *
  1619. * This function is useful for tracers that want to take a "snapshot"
  1620. * of a CPU buffer and has another back up buffer lying around.
  1621. * it is expected that the tracer handles the cpu buffer not being
  1622. * used at the moment.
  1623. */
  1624. int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
  1625. struct ring_buffer *buffer_b, int cpu)
  1626. {
  1627. struct ring_buffer_per_cpu *cpu_buffer_a;
  1628. struct ring_buffer_per_cpu *cpu_buffer_b;
  1629. if (!cpu_isset(cpu, buffer_a->cpumask) ||
  1630. !cpu_isset(cpu, buffer_b->cpumask))
  1631. return -EINVAL;
  1632. /* At least make sure the two buffers are somewhat the same */
  1633. if (buffer_a->size != buffer_b->size ||
  1634. buffer_a->pages != buffer_b->pages)
  1635. return -EINVAL;
  1636. cpu_buffer_a = buffer_a->buffers[cpu];
  1637. cpu_buffer_b = buffer_b->buffers[cpu];
  1638. /*
  1639. * We can't do a synchronize_sched here because this
  1640. * function can be called in atomic context.
  1641. * Normally this will be called from the same CPU as cpu.
  1642. * If not it's up to the caller to protect this.
  1643. */
  1644. atomic_inc(&cpu_buffer_a->record_disabled);
  1645. atomic_inc(&cpu_buffer_b->record_disabled);
  1646. buffer_a->buffers[cpu] = cpu_buffer_b;
  1647. buffer_b->buffers[cpu] = cpu_buffer_a;
  1648. cpu_buffer_b->buffer = buffer_a;
  1649. cpu_buffer_a->buffer = buffer_b;
  1650. atomic_dec(&cpu_buffer_a->record_disabled);
  1651. atomic_dec(&cpu_buffer_b->record_disabled);
  1652. return 0;
  1653. }