builtin-sched.c 43 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921
  1. #include "builtin.h"
  2. #include "perf.h"
  3. #include "util/util.h"
  4. #include "util/cache.h"
  5. #include "util/symbol.h"
  6. #include "util/thread.h"
  7. #include "util/header.h"
  8. #include "util/session.h"
  9. #include "util/parse-options.h"
  10. #include "util/trace-event.h"
  11. #include "util/debug.h"
  12. #include <sys/prctl.h>
  13. #include <semaphore.h>
  14. #include <pthread.h>
  15. #include <math.h>
  16. static char const *input_name = "perf.data";
  17. static char default_sort_order[] = "avg, max, switch, runtime";
  18. static const char *sort_order = default_sort_order;
  19. static int profile_cpu = -1;
  20. #define PR_SET_NAME 15 /* Set process name */
  21. #define MAX_CPUS 4096
  22. static u64 run_measurement_overhead;
  23. static u64 sleep_measurement_overhead;
  24. #define COMM_LEN 20
  25. #define SYM_LEN 129
  26. #define MAX_PID 65536
  27. static unsigned long nr_tasks;
  28. struct sched_atom;
  29. struct task_desc {
  30. unsigned long nr;
  31. unsigned long pid;
  32. char comm[COMM_LEN];
  33. unsigned long nr_events;
  34. unsigned long curr_event;
  35. struct sched_atom **atoms;
  36. pthread_t thread;
  37. sem_t sleep_sem;
  38. sem_t ready_for_work;
  39. sem_t work_done_sem;
  40. u64 cpu_usage;
  41. };
  42. enum sched_event_type {
  43. SCHED_EVENT_RUN,
  44. SCHED_EVENT_SLEEP,
  45. SCHED_EVENT_WAKEUP,
  46. SCHED_EVENT_MIGRATION,
  47. };
  48. struct sched_atom {
  49. enum sched_event_type type;
  50. int specific_wait;
  51. u64 timestamp;
  52. u64 duration;
  53. unsigned long nr;
  54. sem_t *wait_sem;
  55. struct task_desc *wakee;
  56. };
  57. static struct task_desc *pid_to_task[MAX_PID];
  58. static struct task_desc **tasks;
  59. static pthread_mutex_t start_work_mutex = PTHREAD_MUTEX_INITIALIZER;
  60. static u64 start_time;
  61. static pthread_mutex_t work_done_wait_mutex = PTHREAD_MUTEX_INITIALIZER;
  62. static unsigned long nr_run_events;
  63. static unsigned long nr_sleep_events;
  64. static unsigned long nr_wakeup_events;
  65. static unsigned long nr_sleep_corrections;
  66. static unsigned long nr_run_events_optimized;
  67. static unsigned long targetless_wakeups;
  68. static unsigned long multitarget_wakeups;
  69. static u64 cpu_usage;
  70. static u64 runavg_cpu_usage;
  71. static u64 parent_cpu_usage;
  72. static u64 runavg_parent_cpu_usage;
  73. static unsigned long nr_runs;
  74. static u64 sum_runtime;
  75. static u64 sum_fluct;
  76. static u64 run_avg;
  77. static unsigned int replay_repeat = 10;
  78. static unsigned long nr_timestamps;
  79. static unsigned long nr_unordered_timestamps;
  80. static unsigned long nr_state_machine_bugs;
  81. static unsigned long nr_context_switch_bugs;
  82. static unsigned long nr_events;
  83. static unsigned long nr_lost_chunks;
  84. static unsigned long nr_lost_events;
  85. #define TASK_STATE_TO_CHAR_STR "RSDTtZX"
  86. enum thread_state {
  87. THREAD_SLEEPING = 0,
  88. THREAD_WAIT_CPU,
  89. THREAD_SCHED_IN,
  90. THREAD_IGNORE
  91. };
  92. struct work_atom {
  93. struct list_head list;
  94. enum thread_state state;
  95. u64 sched_out_time;
  96. u64 wake_up_time;
  97. u64 sched_in_time;
  98. u64 runtime;
  99. };
  100. struct work_atoms {
  101. struct list_head work_list;
  102. struct thread *thread;
  103. struct rb_node node;
  104. u64 max_lat;
  105. u64 max_lat_at;
  106. u64 total_lat;
  107. u64 nb_atoms;
  108. u64 total_runtime;
  109. };
  110. typedef int (*sort_fn_t)(struct work_atoms *, struct work_atoms *);
  111. static struct rb_root atom_root, sorted_atom_root;
  112. static u64 all_runtime;
  113. static u64 all_count;
  114. static u64 get_nsecs(void)
  115. {
  116. struct timespec ts;
  117. clock_gettime(CLOCK_MONOTONIC, &ts);
  118. return ts.tv_sec * 1000000000ULL + ts.tv_nsec;
  119. }
  120. static void burn_nsecs(u64 nsecs)
  121. {
  122. u64 T0 = get_nsecs(), T1;
  123. do {
  124. T1 = get_nsecs();
  125. } while (T1 + run_measurement_overhead < T0 + nsecs);
  126. }
  127. static void sleep_nsecs(u64 nsecs)
  128. {
  129. struct timespec ts;
  130. ts.tv_nsec = nsecs % 999999999;
  131. ts.tv_sec = nsecs / 999999999;
  132. nanosleep(&ts, NULL);
  133. }
  134. static void calibrate_run_measurement_overhead(void)
  135. {
  136. u64 T0, T1, delta, min_delta = 1000000000ULL;
  137. int i;
  138. for (i = 0; i < 10; i++) {
  139. T0 = get_nsecs();
  140. burn_nsecs(0);
  141. T1 = get_nsecs();
  142. delta = T1-T0;
  143. min_delta = min(min_delta, delta);
  144. }
  145. run_measurement_overhead = min_delta;
  146. printf("run measurement overhead: %Ld nsecs\n", min_delta);
  147. }
  148. static void calibrate_sleep_measurement_overhead(void)
  149. {
  150. u64 T0, T1, delta, min_delta = 1000000000ULL;
  151. int i;
  152. for (i = 0; i < 10; i++) {
  153. T0 = get_nsecs();
  154. sleep_nsecs(10000);
  155. T1 = get_nsecs();
  156. delta = T1-T0;
  157. min_delta = min(min_delta, delta);
  158. }
  159. min_delta -= 10000;
  160. sleep_measurement_overhead = min_delta;
  161. printf("sleep measurement overhead: %Ld nsecs\n", min_delta);
  162. }
  163. static struct sched_atom *
  164. get_new_event(struct task_desc *task, u64 timestamp)
  165. {
  166. struct sched_atom *event = zalloc(sizeof(*event));
  167. unsigned long idx = task->nr_events;
  168. size_t size;
  169. event->timestamp = timestamp;
  170. event->nr = idx;
  171. task->nr_events++;
  172. size = sizeof(struct sched_atom *) * task->nr_events;
  173. task->atoms = realloc(task->atoms, size);
  174. BUG_ON(!task->atoms);
  175. task->atoms[idx] = event;
  176. return event;
  177. }
  178. static struct sched_atom *last_event(struct task_desc *task)
  179. {
  180. if (!task->nr_events)
  181. return NULL;
  182. return task->atoms[task->nr_events - 1];
  183. }
  184. static void
  185. add_sched_event_run(struct task_desc *task, u64 timestamp, u64 duration)
  186. {
  187. struct sched_atom *event, *curr_event = last_event(task);
  188. /*
  189. * optimize an existing RUN event by merging this one
  190. * to it:
  191. */
  192. if (curr_event && curr_event->type == SCHED_EVENT_RUN) {
  193. nr_run_events_optimized++;
  194. curr_event->duration += duration;
  195. return;
  196. }
  197. event = get_new_event(task, timestamp);
  198. event->type = SCHED_EVENT_RUN;
  199. event->duration = duration;
  200. nr_run_events++;
  201. }
  202. static void
  203. add_sched_event_wakeup(struct task_desc *task, u64 timestamp,
  204. struct task_desc *wakee)
  205. {
  206. struct sched_atom *event, *wakee_event;
  207. event = get_new_event(task, timestamp);
  208. event->type = SCHED_EVENT_WAKEUP;
  209. event->wakee = wakee;
  210. wakee_event = last_event(wakee);
  211. if (!wakee_event || wakee_event->type != SCHED_EVENT_SLEEP) {
  212. targetless_wakeups++;
  213. return;
  214. }
  215. if (wakee_event->wait_sem) {
  216. multitarget_wakeups++;
  217. return;
  218. }
  219. wakee_event->wait_sem = zalloc(sizeof(*wakee_event->wait_sem));
  220. sem_init(wakee_event->wait_sem, 0, 0);
  221. wakee_event->specific_wait = 1;
  222. event->wait_sem = wakee_event->wait_sem;
  223. nr_wakeup_events++;
  224. }
  225. static void
  226. add_sched_event_sleep(struct task_desc *task, u64 timestamp,
  227. u64 task_state __used)
  228. {
  229. struct sched_atom *event = get_new_event(task, timestamp);
  230. event->type = SCHED_EVENT_SLEEP;
  231. nr_sleep_events++;
  232. }
  233. static struct task_desc *register_pid(unsigned long pid, const char *comm)
  234. {
  235. struct task_desc *task;
  236. BUG_ON(pid >= MAX_PID);
  237. task = pid_to_task[pid];
  238. if (task)
  239. return task;
  240. task = zalloc(sizeof(*task));
  241. task->pid = pid;
  242. task->nr = nr_tasks;
  243. strcpy(task->comm, comm);
  244. /*
  245. * every task starts in sleeping state - this gets ignored
  246. * if there's no wakeup pointing to this sleep state:
  247. */
  248. add_sched_event_sleep(task, 0, 0);
  249. pid_to_task[pid] = task;
  250. nr_tasks++;
  251. tasks = realloc(tasks, nr_tasks*sizeof(struct task_task *));
  252. BUG_ON(!tasks);
  253. tasks[task->nr] = task;
  254. if (verbose)
  255. printf("registered task #%ld, PID %ld (%s)\n", nr_tasks, pid, comm);
  256. return task;
  257. }
  258. static void print_task_traces(void)
  259. {
  260. struct task_desc *task;
  261. unsigned long i;
  262. for (i = 0; i < nr_tasks; i++) {
  263. task = tasks[i];
  264. printf("task %6ld (%20s:%10ld), nr_events: %ld\n",
  265. task->nr, task->comm, task->pid, task->nr_events);
  266. }
  267. }
  268. static void add_cross_task_wakeups(void)
  269. {
  270. struct task_desc *task1, *task2;
  271. unsigned long i, j;
  272. for (i = 0; i < nr_tasks; i++) {
  273. task1 = tasks[i];
  274. j = i + 1;
  275. if (j == nr_tasks)
  276. j = 0;
  277. task2 = tasks[j];
  278. add_sched_event_wakeup(task1, 0, task2);
  279. }
  280. }
  281. static void
  282. process_sched_event(struct task_desc *this_task __used, struct sched_atom *atom)
  283. {
  284. int ret = 0;
  285. u64 now;
  286. long long delta;
  287. now = get_nsecs();
  288. delta = start_time + atom->timestamp - now;
  289. switch (atom->type) {
  290. case SCHED_EVENT_RUN:
  291. burn_nsecs(atom->duration);
  292. break;
  293. case SCHED_EVENT_SLEEP:
  294. if (atom->wait_sem)
  295. ret = sem_wait(atom->wait_sem);
  296. BUG_ON(ret);
  297. break;
  298. case SCHED_EVENT_WAKEUP:
  299. if (atom->wait_sem)
  300. ret = sem_post(atom->wait_sem);
  301. BUG_ON(ret);
  302. break;
  303. case SCHED_EVENT_MIGRATION:
  304. break;
  305. default:
  306. BUG_ON(1);
  307. }
  308. }
  309. static u64 get_cpu_usage_nsec_parent(void)
  310. {
  311. struct rusage ru;
  312. u64 sum;
  313. int err;
  314. err = getrusage(RUSAGE_SELF, &ru);
  315. BUG_ON(err);
  316. sum = ru.ru_utime.tv_sec*1e9 + ru.ru_utime.tv_usec*1e3;
  317. sum += ru.ru_stime.tv_sec*1e9 + ru.ru_stime.tv_usec*1e3;
  318. return sum;
  319. }
  320. static int self_open_counters(void)
  321. {
  322. struct perf_event_attr attr;
  323. int fd;
  324. memset(&attr, 0, sizeof(attr));
  325. attr.type = PERF_TYPE_SOFTWARE;
  326. attr.config = PERF_COUNT_SW_TASK_CLOCK;
  327. fd = sys_perf_event_open(&attr, 0, -1, -1, 0);
  328. if (fd < 0)
  329. die("Error: sys_perf_event_open() syscall returned"
  330. "with %d (%s)\n", fd, strerror(errno));
  331. return fd;
  332. }
  333. static u64 get_cpu_usage_nsec_self(int fd)
  334. {
  335. u64 runtime;
  336. int ret;
  337. ret = read(fd, &runtime, sizeof(runtime));
  338. BUG_ON(ret != sizeof(runtime));
  339. return runtime;
  340. }
  341. static void *thread_func(void *ctx)
  342. {
  343. struct task_desc *this_task = ctx;
  344. u64 cpu_usage_0, cpu_usage_1;
  345. unsigned long i, ret;
  346. char comm2[22];
  347. int fd;
  348. sprintf(comm2, ":%s", this_task->comm);
  349. prctl(PR_SET_NAME, comm2);
  350. fd = self_open_counters();
  351. again:
  352. ret = sem_post(&this_task->ready_for_work);
  353. BUG_ON(ret);
  354. ret = pthread_mutex_lock(&start_work_mutex);
  355. BUG_ON(ret);
  356. ret = pthread_mutex_unlock(&start_work_mutex);
  357. BUG_ON(ret);
  358. cpu_usage_0 = get_cpu_usage_nsec_self(fd);
  359. for (i = 0; i < this_task->nr_events; i++) {
  360. this_task->curr_event = i;
  361. process_sched_event(this_task, this_task->atoms[i]);
  362. }
  363. cpu_usage_1 = get_cpu_usage_nsec_self(fd);
  364. this_task->cpu_usage = cpu_usage_1 - cpu_usage_0;
  365. ret = sem_post(&this_task->work_done_sem);
  366. BUG_ON(ret);
  367. ret = pthread_mutex_lock(&work_done_wait_mutex);
  368. BUG_ON(ret);
  369. ret = pthread_mutex_unlock(&work_done_wait_mutex);
  370. BUG_ON(ret);
  371. goto again;
  372. }
  373. static void create_tasks(void)
  374. {
  375. struct task_desc *task;
  376. pthread_attr_t attr;
  377. unsigned long i;
  378. int err;
  379. err = pthread_attr_init(&attr);
  380. BUG_ON(err);
  381. err = pthread_attr_setstacksize(&attr,
  382. (size_t) max(16 * 1024, PTHREAD_STACK_MIN));
  383. BUG_ON(err);
  384. err = pthread_mutex_lock(&start_work_mutex);
  385. BUG_ON(err);
  386. err = pthread_mutex_lock(&work_done_wait_mutex);
  387. BUG_ON(err);
  388. for (i = 0; i < nr_tasks; i++) {
  389. task = tasks[i];
  390. sem_init(&task->sleep_sem, 0, 0);
  391. sem_init(&task->ready_for_work, 0, 0);
  392. sem_init(&task->work_done_sem, 0, 0);
  393. task->curr_event = 0;
  394. err = pthread_create(&task->thread, &attr, thread_func, task);
  395. BUG_ON(err);
  396. }
  397. }
  398. static void wait_for_tasks(void)
  399. {
  400. u64 cpu_usage_0, cpu_usage_1;
  401. struct task_desc *task;
  402. unsigned long i, ret;
  403. start_time = get_nsecs();
  404. cpu_usage = 0;
  405. pthread_mutex_unlock(&work_done_wait_mutex);
  406. for (i = 0; i < nr_tasks; i++) {
  407. task = tasks[i];
  408. ret = sem_wait(&task->ready_for_work);
  409. BUG_ON(ret);
  410. sem_init(&task->ready_for_work, 0, 0);
  411. }
  412. ret = pthread_mutex_lock(&work_done_wait_mutex);
  413. BUG_ON(ret);
  414. cpu_usage_0 = get_cpu_usage_nsec_parent();
  415. pthread_mutex_unlock(&start_work_mutex);
  416. for (i = 0; i < nr_tasks; i++) {
  417. task = tasks[i];
  418. ret = sem_wait(&task->work_done_sem);
  419. BUG_ON(ret);
  420. sem_init(&task->work_done_sem, 0, 0);
  421. cpu_usage += task->cpu_usage;
  422. task->cpu_usage = 0;
  423. }
  424. cpu_usage_1 = get_cpu_usage_nsec_parent();
  425. if (!runavg_cpu_usage)
  426. runavg_cpu_usage = cpu_usage;
  427. runavg_cpu_usage = (runavg_cpu_usage*9 + cpu_usage)/10;
  428. parent_cpu_usage = cpu_usage_1 - cpu_usage_0;
  429. if (!runavg_parent_cpu_usage)
  430. runavg_parent_cpu_usage = parent_cpu_usage;
  431. runavg_parent_cpu_usage = (runavg_parent_cpu_usage*9 +
  432. parent_cpu_usage)/10;
  433. ret = pthread_mutex_lock(&start_work_mutex);
  434. BUG_ON(ret);
  435. for (i = 0; i < nr_tasks; i++) {
  436. task = tasks[i];
  437. sem_init(&task->sleep_sem, 0, 0);
  438. task->curr_event = 0;
  439. }
  440. }
  441. static void run_one_test(void)
  442. {
  443. u64 T0, T1, delta, avg_delta, fluct, std_dev;
  444. T0 = get_nsecs();
  445. wait_for_tasks();
  446. T1 = get_nsecs();
  447. delta = T1 - T0;
  448. sum_runtime += delta;
  449. nr_runs++;
  450. avg_delta = sum_runtime / nr_runs;
  451. if (delta < avg_delta)
  452. fluct = avg_delta - delta;
  453. else
  454. fluct = delta - avg_delta;
  455. sum_fluct += fluct;
  456. std_dev = sum_fluct / nr_runs / sqrt(nr_runs);
  457. if (!run_avg)
  458. run_avg = delta;
  459. run_avg = (run_avg*9 + delta)/10;
  460. printf("#%-3ld: %0.3f, ",
  461. nr_runs, (double)delta/1000000.0);
  462. printf("ravg: %0.2f, ",
  463. (double)run_avg/1e6);
  464. printf("cpu: %0.2f / %0.2f",
  465. (double)cpu_usage/1e6, (double)runavg_cpu_usage/1e6);
  466. #if 0
  467. /*
  468. * rusage statistics done by the parent, these are less
  469. * accurate than the sum_exec_runtime based statistics:
  470. */
  471. printf(" [%0.2f / %0.2f]",
  472. (double)parent_cpu_usage/1e6,
  473. (double)runavg_parent_cpu_usage/1e6);
  474. #endif
  475. printf("\n");
  476. if (nr_sleep_corrections)
  477. printf(" (%ld sleep corrections)\n", nr_sleep_corrections);
  478. nr_sleep_corrections = 0;
  479. }
  480. static void test_calibrations(void)
  481. {
  482. u64 T0, T1;
  483. T0 = get_nsecs();
  484. burn_nsecs(1e6);
  485. T1 = get_nsecs();
  486. printf("the run test took %Ld nsecs\n", T1-T0);
  487. T0 = get_nsecs();
  488. sleep_nsecs(1e6);
  489. T1 = get_nsecs();
  490. printf("the sleep test took %Ld nsecs\n", T1-T0);
  491. }
  492. #define FILL_FIELD(ptr, field, event, data) \
  493. ptr.field = (typeof(ptr.field)) raw_field_value(event, #field, data)
  494. #define FILL_ARRAY(ptr, array, event, data) \
  495. do { \
  496. void *__array = raw_field_ptr(event, #array, data); \
  497. memcpy(ptr.array, __array, sizeof(ptr.array)); \
  498. } while(0)
  499. #define FILL_COMMON_FIELDS(ptr, event, data) \
  500. do { \
  501. FILL_FIELD(ptr, common_type, event, data); \
  502. FILL_FIELD(ptr, common_flags, event, data); \
  503. FILL_FIELD(ptr, common_preempt_count, event, data); \
  504. FILL_FIELD(ptr, common_pid, event, data); \
  505. FILL_FIELD(ptr, common_tgid, event, data); \
  506. } while (0)
  507. struct trace_switch_event {
  508. u32 size;
  509. u16 common_type;
  510. u8 common_flags;
  511. u8 common_preempt_count;
  512. u32 common_pid;
  513. u32 common_tgid;
  514. char prev_comm[16];
  515. u32 prev_pid;
  516. u32 prev_prio;
  517. u64 prev_state;
  518. char next_comm[16];
  519. u32 next_pid;
  520. u32 next_prio;
  521. };
  522. struct trace_runtime_event {
  523. u32 size;
  524. u16 common_type;
  525. u8 common_flags;
  526. u8 common_preempt_count;
  527. u32 common_pid;
  528. u32 common_tgid;
  529. char comm[16];
  530. u32 pid;
  531. u64 runtime;
  532. u64 vruntime;
  533. };
  534. struct trace_wakeup_event {
  535. u32 size;
  536. u16 common_type;
  537. u8 common_flags;
  538. u8 common_preempt_count;
  539. u32 common_pid;
  540. u32 common_tgid;
  541. char comm[16];
  542. u32 pid;
  543. u32 prio;
  544. u32 success;
  545. u32 cpu;
  546. };
  547. struct trace_fork_event {
  548. u32 size;
  549. u16 common_type;
  550. u8 common_flags;
  551. u8 common_preempt_count;
  552. u32 common_pid;
  553. u32 common_tgid;
  554. char parent_comm[16];
  555. u32 parent_pid;
  556. char child_comm[16];
  557. u32 child_pid;
  558. };
  559. struct trace_migrate_task_event {
  560. u32 size;
  561. u16 common_type;
  562. u8 common_flags;
  563. u8 common_preempt_count;
  564. u32 common_pid;
  565. u32 common_tgid;
  566. char comm[16];
  567. u32 pid;
  568. u32 prio;
  569. u32 cpu;
  570. };
  571. struct trace_sched_handler {
  572. void (*switch_event)(struct trace_switch_event *,
  573. struct perf_session *,
  574. struct event *,
  575. int cpu,
  576. u64 timestamp,
  577. struct thread *thread);
  578. void (*runtime_event)(struct trace_runtime_event *,
  579. struct perf_session *,
  580. struct event *,
  581. int cpu,
  582. u64 timestamp,
  583. struct thread *thread);
  584. void (*wakeup_event)(struct trace_wakeup_event *,
  585. struct perf_session *,
  586. struct event *,
  587. int cpu,
  588. u64 timestamp,
  589. struct thread *thread);
  590. void (*fork_event)(struct trace_fork_event *,
  591. struct event *,
  592. int cpu,
  593. u64 timestamp,
  594. struct thread *thread);
  595. void (*migrate_task_event)(struct trace_migrate_task_event *,
  596. struct perf_session *session,
  597. struct event *,
  598. int cpu,
  599. u64 timestamp,
  600. struct thread *thread);
  601. };
  602. static void
  603. replay_wakeup_event(struct trace_wakeup_event *wakeup_event,
  604. struct perf_session *session __used,
  605. struct event *event,
  606. int cpu __used,
  607. u64 timestamp __used,
  608. struct thread *thread __used)
  609. {
  610. struct task_desc *waker, *wakee;
  611. if (verbose) {
  612. printf("sched_wakeup event %p\n", event);
  613. printf(" ... pid %d woke up %s/%d\n",
  614. wakeup_event->common_pid,
  615. wakeup_event->comm,
  616. wakeup_event->pid);
  617. }
  618. waker = register_pid(wakeup_event->common_pid, "<unknown>");
  619. wakee = register_pid(wakeup_event->pid, wakeup_event->comm);
  620. add_sched_event_wakeup(waker, timestamp, wakee);
  621. }
  622. static u64 cpu_last_switched[MAX_CPUS];
  623. static void
  624. replay_switch_event(struct trace_switch_event *switch_event,
  625. struct perf_session *session __used,
  626. struct event *event,
  627. int cpu,
  628. u64 timestamp,
  629. struct thread *thread __used)
  630. {
  631. struct task_desc *prev, *next;
  632. u64 timestamp0;
  633. s64 delta;
  634. if (verbose)
  635. printf("sched_switch event %p\n", event);
  636. if (cpu >= MAX_CPUS || cpu < 0)
  637. return;
  638. timestamp0 = cpu_last_switched[cpu];
  639. if (timestamp0)
  640. delta = timestamp - timestamp0;
  641. else
  642. delta = 0;
  643. if (delta < 0)
  644. die("hm, delta: %Ld < 0 ?\n", delta);
  645. if (verbose) {
  646. printf(" ... switch from %s/%d to %s/%d [ran %Ld nsecs]\n",
  647. switch_event->prev_comm, switch_event->prev_pid,
  648. switch_event->next_comm, switch_event->next_pid,
  649. delta);
  650. }
  651. prev = register_pid(switch_event->prev_pid, switch_event->prev_comm);
  652. next = register_pid(switch_event->next_pid, switch_event->next_comm);
  653. cpu_last_switched[cpu] = timestamp;
  654. add_sched_event_run(prev, timestamp, delta);
  655. add_sched_event_sleep(prev, timestamp, switch_event->prev_state);
  656. }
  657. static void
  658. replay_fork_event(struct trace_fork_event *fork_event,
  659. struct event *event,
  660. int cpu __used,
  661. u64 timestamp __used,
  662. struct thread *thread __used)
  663. {
  664. if (verbose) {
  665. printf("sched_fork event %p\n", event);
  666. printf("... parent: %s/%d\n", fork_event->parent_comm, fork_event->parent_pid);
  667. printf("... child: %s/%d\n", fork_event->child_comm, fork_event->child_pid);
  668. }
  669. register_pid(fork_event->parent_pid, fork_event->parent_comm);
  670. register_pid(fork_event->child_pid, fork_event->child_comm);
  671. }
  672. static struct trace_sched_handler replay_ops = {
  673. .wakeup_event = replay_wakeup_event,
  674. .switch_event = replay_switch_event,
  675. .fork_event = replay_fork_event,
  676. };
  677. struct sort_dimension {
  678. const char *name;
  679. sort_fn_t cmp;
  680. struct list_head list;
  681. };
  682. static LIST_HEAD(cmp_pid);
  683. static int
  684. thread_lat_cmp(struct list_head *list, struct work_atoms *l, struct work_atoms *r)
  685. {
  686. struct sort_dimension *sort;
  687. int ret = 0;
  688. BUG_ON(list_empty(list));
  689. list_for_each_entry(sort, list, list) {
  690. ret = sort->cmp(l, r);
  691. if (ret)
  692. return ret;
  693. }
  694. return ret;
  695. }
  696. static struct work_atoms *
  697. thread_atoms_search(struct rb_root *root, struct thread *thread,
  698. struct list_head *sort_list)
  699. {
  700. struct rb_node *node = root->rb_node;
  701. struct work_atoms key = { .thread = thread };
  702. while (node) {
  703. struct work_atoms *atoms;
  704. int cmp;
  705. atoms = container_of(node, struct work_atoms, node);
  706. cmp = thread_lat_cmp(sort_list, &key, atoms);
  707. if (cmp > 0)
  708. node = node->rb_left;
  709. else if (cmp < 0)
  710. node = node->rb_right;
  711. else {
  712. BUG_ON(thread != atoms->thread);
  713. return atoms;
  714. }
  715. }
  716. return NULL;
  717. }
  718. static void
  719. __thread_latency_insert(struct rb_root *root, struct work_atoms *data,
  720. struct list_head *sort_list)
  721. {
  722. struct rb_node **new = &(root->rb_node), *parent = NULL;
  723. while (*new) {
  724. struct work_atoms *this;
  725. int cmp;
  726. this = container_of(*new, struct work_atoms, node);
  727. parent = *new;
  728. cmp = thread_lat_cmp(sort_list, data, this);
  729. if (cmp > 0)
  730. new = &((*new)->rb_left);
  731. else
  732. new = &((*new)->rb_right);
  733. }
  734. rb_link_node(&data->node, parent, new);
  735. rb_insert_color(&data->node, root);
  736. }
  737. static void thread_atoms_insert(struct thread *thread)
  738. {
  739. struct work_atoms *atoms = zalloc(sizeof(*atoms));
  740. if (!atoms)
  741. die("No memory");
  742. atoms->thread = thread;
  743. INIT_LIST_HEAD(&atoms->work_list);
  744. __thread_latency_insert(&atom_root, atoms, &cmp_pid);
  745. }
  746. static void
  747. latency_fork_event(struct trace_fork_event *fork_event __used,
  748. struct event *event __used,
  749. int cpu __used,
  750. u64 timestamp __used,
  751. struct thread *thread __used)
  752. {
  753. /* should insert the newcomer */
  754. }
  755. __used
  756. static char sched_out_state(struct trace_switch_event *switch_event)
  757. {
  758. const char *str = TASK_STATE_TO_CHAR_STR;
  759. return str[switch_event->prev_state];
  760. }
  761. static void
  762. add_sched_out_event(struct work_atoms *atoms,
  763. char run_state,
  764. u64 timestamp)
  765. {
  766. struct work_atom *atom = zalloc(sizeof(*atom));
  767. if (!atom)
  768. die("Non memory");
  769. atom->sched_out_time = timestamp;
  770. if (run_state == 'R') {
  771. atom->state = THREAD_WAIT_CPU;
  772. atom->wake_up_time = atom->sched_out_time;
  773. }
  774. list_add_tail(&atom->list, &atoms->work_list);
  775. }
  776. static void
  777. add_runtime_event(struct work_atoms *atoms, u64 delta, u64 timestamp __used)
  778. {
  779. struct work_atom *atom;
  780. BUG_ON(list_empty(&atoms->work_list));
  781. atom = list_entry(atoms->work_list.prev, struct work_atom, list);
  782. atom->runtime += delta;
  783. atoms->total_runtime += delta;
  784. }
  785. static void
  786. add_sched_in_event(struct work_atoms *atoms, u64 timestamp)
  787. {
  788. struct work_atom *atom;
  789. u64 delta;
  790. if (list_empty(&atoms->work_list))
  791. return;
  792. atom = list_entry(atoms->work_list.prev, struct work_atom, list);
  793. if (atom->state != THREAD_WAIT_CPU)
  794. return;
  795. if (timestamp < atom->wake_up_time) {
  796. atom->state = THREAD_IGNORE;
  797. return;
  798. }
  799. atom->state = THREAD_SCHED_IN;
  800. atom->sched_in_time = timestamp;
  801. delta = atom->sched_in_time - atom->wake_up_time;
  802. atoms->total_lat += delta;
  803. if (delta > atoms->max_lat) {
  804. atoms->max_lat = delta;
  805. atoms->max_lat_at = timestamp;
  806. }
  807. atoms->nb_atoms++;
  808. }
  809. static void
  810. latency_switch_event(struct trace_switch_event *switch_event,
  811. struct perf_session *session,
  812. struct event *event __used,
  813. int cpu,
  814. u64 timestamp,
  815. struct thread *thread __used)
  816. {
  817. struct work_atoms *out_events, *in_events;
  818. struct thread *sched_out, *sched_in;
  819. u64 timestamp0;
  820. s64 delta;
  821. BUG_ON(cpu >= MAX_CPUS || cpu < 0);
  822. timestamp0 = cpu_last_switched[cpu];
  823. cpu_last_switched[cpu] = timestamp;
  824. if (timestamp0)
  825. delta = timestamp - timestamp0;
  826. else
  827. delta = 0;
  828. if (delta < 0)
  829. die("hm, delta: %Ld < 0 ?\n", delta);
  830. sched_out = perf_session__findnew(session, switch_event->prev_pid);
  831. sched_in = perf_session__findnew(session, switch_event->next_pid);
  832. out_events = thread_atoms_search(&atom_root, sched_out, &cmp_pid);
  833. if (!out_events) {
  834. thread_atoms_insert(sched_out);
  835. out_events = thread_atoms_search(&atom_root, sched_out, &cmp_pid);
  836. if (!out_events)
  837. die("out-event: Internal tree error");
  838. }
  839. add_sched_out_event(out_events, sched_out_state(switch_event), timestamp);
  840. in_events = thread_atoms_search(&atom_root, sched_in, &cmp_pid);
  841. if (!in_events) {
  842. thread_atoms_insert(sched_in);
  843. in_events = thread_atoms_search(&atom_root, sched_in, &cmp_pid);
  844. if (!in_events)
  845. die("in-event: Internal tree error");
  846. /*
  847. * Take came in we have not heard about yet,
  848. * add in an initial atom in runnable state:
  849. */
  850. add_sched_out_event(in_events, 'R', timestamp);
  851. }
  852. add_sched_in_event(in_events, timestamp);
  853. }
  854. static void
  855. latency_runtime_event(struct trace_runtime_event *runtime_event,
  856. struct perf_session *session,
  857. struct event *event __used,
  858. int cpu,
  859. u64 timestamp,
  860. struct thread *this_thread __used)
  861. {
  862. struct thread *thread = perf_session__findnew(session, runtime_event->pid);
  863. struct work_atoms *atoms = thread_atoms_search(&atom_root, thread, &cmp_pid);
  864. BUG_ON(cpu >= MAX_CPUS || cpu < 0);
  865. if (!atoms) {
  866. thread_atoms_insert(thread);
  867. atoms = thread_atoms_search(&atom_root, thread, &cmp_pid);
  868. if (!atoms)
  869. die("in-event: Internal tree error");
  870. add_sched_out_event(atoms, 'R', timestamp);
  871. }
  872. add_runtime_event(atoms, runtime_event->runtime, timestamp);
  873. }
  874. static void
  875. latency_wakeup_event(struct trace_wakeup_event *wakeup_event,
  876. struct perf_session *session,
  877. struct event *__event __used,
  878. int cpu __used,
  879. u64 timestamp,
  880. struct thread *thread __used)
  881. {
  882. struct work_atoms *atoms;
  883. struct work_atom *atom;
  884. struct thread *wakee;
  885. /* Note for later, it may be interesting to observe the failing cases */
  886. if (!wakeup_event->success)
  887. return;
  888. wakee = perf_session__findnew(session, wakeup_event->pid);
  889. atoms = thread_atoms_search(&atom_root, wakee, &cmp_pid);
  890. if (!atoms) {
  891. thread_atoms_insert(wakee);
  892. atoms = thread_atoms_search(&atom_root, wakee, &cmp_pid);
  893. if (!atoms)
  894. die("wakeup-event: Internal tree error");
  895. add_sched_out_event(atoms, 'S', timestamp);
  896. }
  897. BUG_ON(list_empty(&atoms->work_list));
  898. atom = list_entry(atoms->work_list.prev, struct work_atom, list);
  899. /*
  900. * You WILL be missing events if you've recorded only
  901. * one CPU, or are only looking at only one, so don't
  902. * make useless noise.
  903. */
  904. if (profile_cpu == -1 && atom->state != THREAD_SLEEPING)
  905. nr_state_machine_bugs++;
  906. nr_timestamps++;
  907. if (atom->sched_out_time > timestamp) {
  908. nr_unordered_timestamps++;
  909. return;
  910. }
  911. atom->state = THREAD_WAIT_CPU;
  912. atom->wake_up_time = timestamp;
  913. }
  914. static void
  915. latency_migrate_task_event(struct trace_migrate_task_event *migrate_task_event,
  916. struct perf_session *session,
  917. struct event *__event __used,
  918. int cpu __used,
  919. u64 timestamp,
  920. struct thread *thread __used)
  921. {
  922. struct work_atoms *atoms;
  923. struct work_atom *atom;
  924. struct thread *migrant;
  925. /*
  926. * Only need to worry about migration when profiling one CPU.
  927. */
  928. if (profile_cpu == -1)
  929. return;
  930. migrant = perf_session__findnew(session, migrate_task_event->pid);
  931. atoms = thread_atoms_search(&atom_root, migrant, &cmp_pid);
  932. if (!atoms) {
  933. thread_atoms_insert(migrant);
  934. register_pid(migrant->pid, migrant->comm);
  935. atoms = thread_atoms_search(&atom_root, migrant, &cmp_pid);
  936. if (!atoms)
  937. die("migration-event: Internal tree error");
  938. add_sched_out_event(atoms, 'R', timestamp);
  939. }
  940. BUG_ON(list_empty(&atoms->work_list));
  941. atom = list_entry(atoms->work_list.prev, struct work_atom, list);
  942. atom->sched_in_time = atom->sched_out_time = atom->wake_up_time = timestamp;
  943. nr_timestamps++;
  944. if (atom->sched_out_time > timestamp)
  945. nr_unordered_timestamps++;
  946. }
  947. static struct trace_sched_handler lat_ops = {
  948. .wakeup_event = latency_wakeup_event,
  949. .switch_event = latency_switch_event,
  950. .runtime_event = latency_runtime_event,
  951. .fork_event = latency_fork_event,
  952. .migrate_task_event = latency_migrate_task_event,
  953. };
  954. static void output_lat_thread(struct work_atoms *work_list)
  955. {
  956. int i;
  957. int ret;
  958. u64 avg;
  959. if (!work_list->nb_atoms)
  960. return;
  961. /*
  962. * Ignore idle threads:
  963. */
  964. if (!strcmp(work_list->thread->comm, "swapper"))
  965. return;
  966. all_runtime += work_list->total_runtime;
  967. all_count += work_list->nb_atoms;
  968. ret = printf(" %s:%d ", work_list->thread->comm, work_list->thread->pid);
  969. for (i = 0; i < 24 - ret; i++)
  970. printf(" ");
  971. avg = work_list->total_lat / work_list->nb_atoms;
  972. printf("|%11.3f ms |%9llu | avg:%9.3f ms | max:%9.3f ms | max at: %9.6f s\n",
  973. (double)work_list->total_runtime / 1e6,
  974. work_list->nb_atoms, (double)avg / 1e6,
  975. (double)work_list->max_lat / 1e6,
  976. (double)work_list->max_lat_at / 1e9);
  977. }
  978. static int pid_cmp(struct work_atoms *l, struct work_atoms *r)
  979. {
  980. if (l->thread->pid < r->thread->pid)
  981. return -1;
  982. if (l->thread->pid > r->thread->pid)
  983. return 1;
  984. return 0;
  985. }
  986. static struct sort_dimension pid_sort_dimension = {
  987. .name = "pid",
  988. .cmp = pid_cmp,
  989. };
  990. static int avg_cmp(struct work_atoms *l, struct work_atoms *r)
  991. {
  992. u64 avgl, avgr;
  993. if (!l->nb_atoms)
  994. return -1;
  995. if (!r->nb_atoms)
  996. return 1;
  997. avgl = l->total_lat / l->nb_atoms;
  998. avgr = r->total_lat / r->nb_atoms;
  999. if (avgl < avgr)
  1000. return -1;
  1001. if (avgl > avgr)
  1002. return 1;
  1003. return 0;
  1004. }
  1005. static struct sort_dimension avg_sort_dimension = {
  1006. .name = "avg",
  1007. .cmp = avg_cmp,
  1008. };
  1009. static int max_cmp(struct work_atoms *l, struct work_atoms *r)
  1010. {
  1011. if (l->max_lat < r->max_lat)
  1012. return -1;
  1013. if (l->max_lat > r->max_lat)
  1014. return 1;
  1015. return 0;
  1016. }
  1017. static struct sort_dimension max_sort_dimension = {
  1018. .name = "max",
  1019. .cmp = max_cmp,
  1020. };
  1021. static int switch_cmp(struct work_atoms *l, struct work_atoms *r)
  1022. {
  1023. if (l->nb_atoms < r->nb_atoms)
  1024. return -1;
  1025. if (l->nb_atoms > r->nb_atoms)
  1026. return 1;
  1027. return 0;
  1028. }
  1029. static struct sort_dimension switch_sort_dimension = {
  1030. .name = "switch",
  1031. .cmp = switch_cmp,
  1032. };
  1033. static int runtime_cmp(struct work_atoms *l, struct work_atoms *r)
  1034. {
  1035. if (l->total_runtime < r->total_runtime)
  1036. return -1;
  1037. if (l->total_runtime > r->total_runtime)
  1038. return 1;
  1039. return 0;
  1040. }
  1041. static struct sort_dimension runtime_sort_dimension = {
  1042. .name = "runtime",
  1043. .cmp = runtime_cmp,
  1044. };
  1045. static struct sort_dimension *available_sorts[] = {
  1046. &pid_sort_dimension,
  1047. &avg_sort_dimension,
  1048. &max_sort_dimension,
  1049. &switch_sort_dimension,
  1050. &runtime_sort_dimension,
  1051. };
  1052. #define NB_AVAILABLE_SORTS (int)(sizeof(available_sorts) / sizeof(struct sort_dimension *))
  1053. static LIST_HEAD(sort_list);
  1054. static int sort_dimension__add(const char *tok, struct list_head *list)
  1055. {
  1056. int i;
  1057. for (i = 0; i < NB_AVAILABLE_SORTS; i++) {
  1058. if (!strcmp(available_sorts[i]->name, tok)) {
  1059. list_add_tail(&available_sorts[i]->list, list);
  1060. return 0;
  1061. }
  1062. }
  1063. return -1;
  1064. }
  1065. static void setup_sorting(void);
  1066. static void sort_lat(void)
  1067. {
  1068. struct rb_node *node;
  1069. for (;;) {
  1070. struct work_atoms *data;
  1071. node = rb_first(&atom_root);
  1072. if (!node)
  1073. break;
  1074. rb_erase(node, &atom_root);
  1075. data = rb_entry(node, struct work_atoms, node);
  1076. __thread_latency_insert(&sorted_atom_root, data, &sort_list);
  1077. }
  1078. }
  1079. static struct trace_sched_handler *trace_handler;
  1080. static void
  1081. process_sched_wakeup_event(void *data, struct perf_session *session,
  1082. struct event *event,
  1083. int cpu __used,
  1084. u64 timestamp __used,
  1085. struct thread *thread __used)
  1086. {
  1087. struct trace_wakeup_event wakeup_event;
  1088. FILL_COMMON_FIELDS(wakeup_event, event, data);
  1089. FILL_ARRAY(wakeup_event, comm, event, data);
  1090. FILL_FIELD(wakeup_event, pid, event, data);
  1091. FILL_FIELD(wakeup_event, prio, event, data);
  1092. FILL_FIELD(wakeup_event, success, event, data);
  1093. FILL_FIELD(wakeup_event, cpu, event, data);
  1094. if (trace_handler->wakeup_event)
  1095. trace_handler->wakeup_event(&wakeup_event, session, event,
  1096. cpu, timestamp, thread);
  1097. }
  1098. /*
  1099. * Track the current task - that way we can know whether there's any
  1100. * weird events, such as a task being switched away that is not current.
  1101. */
  1102. static int max_cpu;
  1103. static u32 curr_pid[MAX_CPUS] = { [0 ... MAX_CPUS-1] = -1 };
  1104. static struct thread *curr_thread[MAX_CPUS];
  1105. static char next_shortname1 = 'A';
  1106. static char next_shortname2 = '0';
  1107. static void
  1108. map_switch_event(struct trace_switch_event *switch_event,
  1109. struct perf_session *session,
  1110. struct event *event __used,
  1111. int this_cpu,
  1112. u64 timestamp,
  1113. struct thread *thread __used)
  1114. {
  1115. struct thread *sched_out, *sched_in;
  1116. int new_shortname;
  1117. u64 timestamp0;
  1118. s64 delta;
  1119. int cpu;
  1120. BUG_ON(this_cpu >= MAX_CPUS || this_cpu < 0);
  1121. if (this_cpu > max_cpu)
  1122. max_cpu = this_cpu;
  1123. timestamp0 = cpu_last_switched[this_cpu];
  1124. cpu_last_switched[this_cpu] = timestamp;
  1125. if (timestamp0)
  1126. delta = timestamp - timestamp0;
  1127. else
  1128. delta = 0;
  1129. if (delta < 0)
  1130. die("hm, delta: %Ld < 0 ?\n", delta);
  1131. sched_out = perf_session__findnew(session, switch_event->prev_pid);
  1132. sched_in = perf_session__findnew(session, switch_event->next_pid);
  1133. curr_thread[this_cpu] = sched_in;
  1134. printf(" ");
  1135. new_shortname = 0;
  1136. if (!sched_in->shortname[0]) {
  1137. sched_in->shortname[0] = next_shortname1;
  1138. sched_in->shortname[1] = next_shortname2;
  1139. if (next_shortname1 < 'Z') {
  1140. next_shortname1++;
  1141. } else {
  1142. next_shortname1='A';
  1143. if (next_shortname2 < '9') {
  1144. next_shortname2++;
  1145. } else {
  1146. next_shortname2='0';
  1147. }
  1148. }
  1149. new_shortname = 1;
  1150. }
  1151. for (cpu = 0; cpu <= max_cpu; cpu++) {
  1152. if (cpu != this_cpu)
  1153. printf(" ");
  1154. else
  1155. printf("*");
  1156. if (curr_thread[cpu]) {
  1157. if (curr_thread[cpu]->pid)
  1158. printf("%2s ", curr_thread[cpu]->shortname);
  1159. else
  1160. printf(". ");
  1161. } else
  1162. printf(" ");
  1163. }
  1164. printf(" %12.6f secs ", (double)timestamp/1e9);
  1165. if (new_shortname) {
  1166. printf("%s => %s:%d\n",
  1167. sched_in->shortname, sched_in->comm, sched_in->pid);
  1168. } else {
  1169. printf("\n");
  1170. }
  1171. }
  1172. static void
  1173. process_sched_switch_event(void *data, struct perf_session *session,
  1174. struct event *event,
  1175. int this_cpu,
  1176. u64 timestamp __used,
  1177. struct thread *thread __used)
  1178. {
  1179. struct trace_switch_event switch_event;
  1180. FILL_COMMON_FIELDS(switch_event, event, data);
  1181. FILL_ARRAY(switch_event, prev_comm, event, data);
  1182. FILL_FIELD(switch_event, prev_pid, event, data);
  1183. FILL_FIELD(switch_event, prev_prio, event, data);
  1184. FILL_FIELD(switch_event, prev_state, event, data);
  1185. FILL_ARRAY(switch_event, next_comm, event, data);
  1186. FILL_FIELD(switch_event, next_pid, event, data);
  1187. FILL_FIELD(switch_event, next_prio, event, data);
  1188. if (curr_pid[this_cpu] != (u32)-1) {
  1189. /*
  1190. * Are we trying to switch away a PID that is
  1191. * not current?
  1192. */
  1193. if (curr_pid[this_cpu] != switch_event.prev_pid)
  1194. nr_context_switch_bugs++;
  1195. }
  1196. if (trace_handler->switch_event)
  1197. trace_handler->switch_event(&switch_event, session, event,
  1198. this_cpu, timestamp, thread);
  1199. curr_pid[this_cpu] = switch_event.next_pid;
  1200. }
  1201. static void
  1202. process_sched_runtime_event(void *data, struct perf_session *session,
  1203. struct event *event,
  1204. int cpu __used,
  1205. u64 timestamp __used,
  1206. struct thread *thread __used)
  1207. {
  1208. struct trace_runtime_event runtime_event;
  1209. FILL_ARRAY(runtime_event, comm, event, data);
  1210. FILL_FIELD(runtime_event, pid, event, data);
  1211. FILL_FIELD(runtime_event, runtime, event, data);
  1212. FILL_FIELD(runtime_event, vruntime, event, data);
  1213. if (trace_handler->runtime_event)
  1214. trace_handler->runtime_event(&runtime_event, session, event, cpu, timestamp, thread);
  1215. }
  1216. static void
  1217. process_sched_fork_event(void *data,
  1218. struct event *event,
  1219. int cpu __used,
  1220. u64 timestamp __used,
  1221. struct thread *thread __used)
  1222. {
  1223. struct trace_fork_event fork_event;
  1224. FILL_COMMON_FIELDS(fork_event, event, data);
  1225. FILL_ARRAY(fork_event, parent_comm, event, data);
  1226. FILL_FIELD(fork_event, parent_pid, event, data);
  1227. FILL_ARRAY(fork_event, child_comm, event, data);
  1228. FILL_FIELD(fork_event, child_pid, event, data);
  1229. if (trace_handler->fork_event)
  1230. trace_handler->fork_event(&fork_event, event,
  1231. cpu, timestamp, thread);
  1232. }
  1233. static void
  1234. process_sched_exit_event(struct event *event,
  1235. int cpu __used,
  1236. u64 timestamp __used,
  1237. struct thread *thread __used)
  1238. {
  1239. if (verbose)
  1240. printf("sched_exit event %p\n", event);
  1241. }
  1242. static void
  1243. process_sched_migrate_task_event(void *data, struct perf_session *session,
  1244. struct event *event,
  1245. int cpu __used,
  1246. u64 timestamp __used,
  1247. struct thread *thread __used)
  1248. {
  1249. struct trace_migrate_task_event migrate_task_event;
  1250. FILL_COMMON_FIELDS(migrate_task_event, event, data);
  1251. FILL_ARRAY(migrate_task_event, comm, event, data);
  1252. FILL_FIELD(migrate_task_event, pid, event, data);
  1253. FILL_FIELD(migrate_task_event, prio, event, data);
  1254. FILL_FIELD(migrate_task_event, cpu, event, data);
  1255. if (trace_handler->migrate_task_event)
  1256. trace_handler->migrate_task_event(&migrate_task_event, session,
  1257. event, cpu, timestamp, thread);
  1258. }
  1259. static void
  1260. process_raw_event(event_t *raw_event __used, struct perf_session *session,
  1261. void *data, int cpu, u64 timestamp, struct thread *thread)
  1262. {
  1263. struct event *event;
  1264. int type;
  1265. type = trace_parse_common_type(data);
  1266. event = trace_find_event(type);
  1267. if (!strcmp(event->name, "sched_switch"))
  1268. process_sched_switch_event(data, session, event, cpu, timestamp, thread);
  1269. if (!strcmp(event->name, "sched_stat_runtime"))
  1270. process_sched_runtime_event(data, session, event, cpu, timestamp, thread);
  1271. if (!strcmp(event->name, "sched_wakeup"))
  1272. process_sched_wakeup_event(data, session, event, cpu, timestamp, thread);
  1273. if (!strcmp(event->name, "sched_wakeup_new"))
  1274. process_sched_wakeup_event(data, session, event, cpu, timestamp, thread);
  1275. if (!strcmp(event->name, "sched_process_fork"))
  1276. process_sched_fork_event(data, event, cpu, timestamp, thread);
  1277. if (!strcmp(event->name, "sched_process_exit"))
  1278. process_sched_exit_event(event, cpu, timestamp, thread);
  1279. if (!strcmp(event->name, "sched_migrate_task"))
  1280. process_sched_migrate_task_event(data, session, event, cpu, timestamp, thread);
  1281. }
  1282. static int process_sample_event(event_t *event, struct sample_data *sample,
  1283. struct perf_session *session)
  1284. {
  1285. struct thread *thread;
  1286. if (!(session->sample_type & PERF_SAMPLE_RAW))
  1287. return 0;
  1288. thread = perf_session__findnew(session, sample->pid);
  1289. if (thread == NULL) {
  1290. pr_debug("problem processing %d event, skipping it.\n",
  1291. event->header.type);
  1292. return -1;
  1293. }
  1294. dump_printf(" ... thread: %s:%d\n", thread->comm, thread->pid);
  1295. if (profile_cpu != -1 && profile_cpu != (int)sample->cpu)
  1296. return 0;
  1297. process_raw_event(event, session, sample->raw_data, sample->cpu,
  1298. sample->time, thread);
  1299. return 0;
  1300. }
  1301. static struct perf_event_ops event_ops = {
  1302. .sample = process_sample_event,
  1303. .comm = event__process_comm,
  1304. .lost = event__process_lost,
  1305. .fork = event__process_task,
  1306. .ordered_samples = true,
  1307. };
  1308. static int read_events(void)
  1309. {
  1310. int err = -EINVAL;
  1311. struct perf_session *session = perf_session__new(input_name, O_RDONLY,
  1312. 0, false, &event_ops);
  1313. if (session == NULL)
  1314. return -ENOMEM;
  1315. if (perf_session__has_traces(session, "record -R")) {
  1316. err = perf_session__process_events(session, &event_ops);
  1317. nr_events = session->hists.stats.nr_events[0];
  1318. nr_lost_events = session->hists.stats.total_lost;
  1319. nr_lost_chunks = session->hists.stats.nr_events[PERF_RECORD_LOST];
  1320. }
  1321. perf_session__delete(session);
  1322. return err;
  1323. }
  1324. static void print_bad_events(void)
  1325. {
  1326. if (nr_unordered_timestamps && nr_timestamps) {
  1327. printf(" INFO: %.3f%% unordered timestamps (%ld out of %ld)\n",
  1328. (double)nr_unordered_timestamps/(double)nr_timestamps*100.0,
  1329. nr_unordered_timestamps, nr_timestamps);
  1330. }
  1331. if (nr_lost_events && nr_events) {
  1332. printf(" INFO: %.3f%% lost events (%ld out of %ld, in %ld chunks)\n",
  1333. (double)nr_lost_events/(double)nr_events*100.0,
  1334. nr_lost_events, nr_events, nr_lost_chunks);
  1335. }
  1336. if (nr_state_machine_bugs && nr_timestamps) {
  1337. printf(" INFO: %.3f%% state machine bugs (%ld out of %ld)",
  1338. (double)nr_state_machine_bugs/(double)nr_timestamps*100.0,
  1339. nr_state_machine_bugs, nr_timestamps);
  1340. if (nr_lost_events)
  1341. printf(" (due to lost events?)");
  1342. printf("\n");
  1343. }
  1344. if (nr_context_switch_bugs && nr_timestamps) {
  1345. printf(" INFO: %.3f%% context switch bugs (%ld out of %ld)",
  1346. (double)nr_context_switch_bugs/(double)nr_timestamps*100.0,
  1347. nr_context_switch_bugs, nr_timestamps);
  1348. if (nr_lost_events)
  1349. printf(" (due to lost events?)");
  1350. printf("\n");
  1351. }
  1352. }
  1353. static void __cmd_lat(void)
  1354. {
  1355. struct rb_node *next;
  1356. setup_pager();
  1357. read_events();
  1358. sort_lat();
  1359. printf("\n ---------------------------------------------------------------------------------------------------------------\n");
  1360. printf(" Task | Runtime ms | Switches | Average delay ms | Maximum delay ms | Maximum delay at |\n");
  1361. printf(" ---------------------------------------------------------------------------------------------------------------\n");
  1362. next = rb_first(&sorted_atom_root);
  1363. while (next) {
  1364. struct work_atoms *work_list;
  1365. work_list = rb_entry(next, struct work_atoms, node);
  1366. output_lat_thread(work_list);
  1367. next = rb_next(next);
  1368. }
  1369. printf(" -----------------------------------------------------------------------------------------\n");
  1370. printf(" TOTAL: |%11.3f ms |%9Ld |\n",
  1371. (double)all_runtime/1e6, all_count);
  1372. printf(" ---------------------------------------------------\n");
  1373. print_bad_events();
  1374. printf("\n");
  1375. }
  1376. static struct trace_sched_handler map_ops = {
  1377. .wakeup_event = NULL,
  1378. .switch_event = map_switch_event,
  1379. .runtime_event = NULL,
  1380. .fork_event = NULL,
  1381. };
  1382. static void __cmd_map(void)
  1383. {
  1384. max_cpu = sysconf(_SC_NPROCESSORS_CONF);
  1385. setup_pager();
  1386. read_events();
  1387. print_bad_events();
  1388. }
  1389. static void __cmd_replay(void)
  1390. {
  1391. unsigned long i;
  1392. calibrate_run_measurement_overhead();
  1393. calibrate_sleep_measurement_overhead();
  1394. test_calibrations();
  1395. read_events();
  1396. printf("nr_run_events: %ld\n", nr_run_events);
  1397. printf("nr_sleep_events: %ld\n", nr_sleep_events);
  1398. printf("nr_wakeup_events: %ld\n", nr_wakeup_events);
  1399. if (targetless_wakeups)
  1400. printf("target-less wakeups: %ld\n", targetless_wakeups);
  1401. if (multitarget_wakeups)
  1402. printf("multi-target wakeups: %ld\n", multitarget_wakeups);
  1403. if (nr_run_events_optimized)
  1404. printf("run atoms optimized: %ld\n",
  1405. nr_run_events_optimized);
  1406. print_task_traces();
  1407. add_cross_task_wakeups();
  1408. create_tasks();
  1409. printf("------------------------------------------------------------\n");
  1410. for (i = 0; i < replay_repeat; i++)
  1411. run_one_test();
  1412. }
  1413. static const char * const sched_usage[] = {
  1414. "perf sched [<options>] {record|latency|map|replay|trace}",
  1415. NULL
  1416. };
  1417. static const struct option sched_options[] = {
  1418. OPT_STRING('i', "input", &input_name, "file",
  1419. "input file name"),
  1420. OPT_INCR('v', "verbose", &verbose,
  1421. "be more verbose (show symbol address, etc)"),
  1422. OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
  1423. "dump raw trace in ASCII"),
  1424. OPT_END()
  1425. };
  1426. static const char * const latency_usage[] = {
  1427. "perf sched latency [<options>]",
  1428. NULL
  1429. };
  1430. static const struct option latency_options[] = {
  1431. OPT_STRING('s', "sort", &sort_order, "key[,key2...]",
  1432. "sort by key(s): runtime, switch, avg, max"),
  1433. OPT_INCR('v', "verbose", &verbose,
  1434. "be more verbose (show symbol address, etc)"),
  1435. OPT_INTEGER('C', "CPU", &profile_cpu,
  1436. "CPU to profile on"),
  1437. OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
  1438. "dump raw trace in ASCII"),
  1439. OPT_END()
  1440. };
  1441. static const char * const replay_usage[] = {
  1442. "perf sched replay [<options>]",
  1443. NULL
  1444. };
  1445. static const struct option replay_options[] = {
  1446. OPT_UINTEGER('r', "repeat", &replay_repeat,
  1447. "repeat the workload replay N times (-1: infinite)"),
  1448. OPT_INCR('v', "verbose", &verbose,
  1449. "be more verbose (show symbol address, etc)"),
  1450. OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
  1451. "dump raw trace in ASCII"),
  1452. OPT_END()
  1453. };
  1454. static void setup_sorting(void)
  1455. {
  1456. char *tmp, *tok, *str = strdup(sort_order);
  1457. for (tok = strtok_r(str, ", ", &tmp);
  1458. tok; tok = strtok_r(NULL, ", ", &tmp)) {
  1459. if (sort_dimension__add(tok, &sort_list) < 0) {
  1460. error("Unknown --sort key: `%s'", tok);
  1461. usage_with_options(latency_usage, latency_options);
  1462. }
  1463. }
  1464. free(str);
  1465. sort_dimension__add("pid", &cmp_pid);
  1466. }
  1467. static const char *record_args[] = {
  1468. "record",
  1469. "-a",
  1470. "-R",
  1471. "-f",
  1472. "-m", "1024",
  1473. "-c", "1",
  1474. "-e", "sched:sched_switch:r",
  1475. "-e", "sched:sched_stat_wait:r",
  1476. "-e", "sched:sched_stat_sleep:r",
  1477. "-e", "sched:sched_stat_iowait:r",
  1478. "-e", "sched:sched_stat_runtime:r",
  1479. "-e", "sched:sched_process_exit:r",
  1480. "-e", "sched:sched_process_fork:r",
  1481. "-e", "sched:sched_wakeup:r",
  1482. "-e", "sched:sched_migrate_task:r",
  1483. };
  1484. static int __cmd_record(int argc, const char **argv)
  1485. {
  1486. unsigned int rec_argc, i, j;
  1487. const char **rec_argv;
  1488. rec_argc = ARRAY_SIZE(record_args) + argc - 1;
  1489. rec_argv = calloc(rec_argc + 1, sizeof(char *));
  1490. if (rec_argv == NULL)
  1491. return -ENOMEM;
  1492. for (i = 0; i < ARRAY_SIZE(record_args); i++)
  1493. rec_argv[i] = strdup(record_args[i]);
  1494. for (j = 1; j < (unsigned int)argc; j++, i++)
  1495. rec_argv[i] = argv[j];
  1496. BUG_ON(i != rec_argc);
  1497. return cmd_record(i, rec_argv, NULL);
  1498. }
  1499. int cmd_sched(int argc, const char **argv, const char *prefix __used)
  1500. {
  1501. argc = parse_options(argc, argv, sched_options, sched_usage,
  1502. PARSE_OPT_STOP_AT_NON_OPTION);
  1503. if (!argc)
  1504. usage_with_options(sched_usage, sched_options);
  1505. /*
  1506. * Aliased to 'perf script' for now:
  1507. */
  1508. if (!strcmp(argv[0], "script"))
  1509. return cmd_script(argc, argv, prefix);
  1510. symbol__init();
  1511. if (!strncmp(argv[0], "rec", 3)) {
  1512. return __cmd_record(argc, argv);
  1513. } else if (!strncmp(argv[0], "lat", 3)) {
  1514. trace_handler = &lat_ops;
  1515. if (argc > 1) {
  1516. argc = parse_options(argc, argv, latency_options, latency_usage, 0);
  1517. if (argc)
  1518. usage_with_options(latency_usage, latency_options);
  1519. }
  1520. setup_sorting();
  1521. __cmd_lat();
  1522. } else if (!strcmp(argv[0], "map")) {
  1523. trace_handler = &map_ops;
  1524. setup_sorting();
  1525. __cmd_map();
  1526. } else if (!strncmp(argv[0], "rep", 3)) {
  1527. trace_handler = &replay_ops;
  1528. if (argc) {
  1529. argc = parse_options(argc, argv, replay_options, replay_usage, 0);
  1530. if (argc)
  1531. usage_with_options(replay_usage, replay_options);
  1532. }
  1533. __cmd_replay();
  1534. } else {
  1535. usage_with_options(sched_usage, sched_options);
  1536. }
  1537. return 0;
  1538. }