task_mmu.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862
  1. #include <linux/mm.h>
  2. #include <linux/hugetlb.h>
  3. #include <linux/mount.h>
  4. #include <linux/seq_file.h>
  5. #include <linux/highmem.h>
  6. #include <linux/ptrace.h>
  7. #include <linux/pagemap.h>
  8. #include <linux/mempolicy.h>
  9. #include <linux/swap.h>
  10. #include <linux/swapops.h>
  11. #include <asm/elf.h>
  12. #include <asm/uaccess.h>
  13. #include <asm/tlbflush.h>
  14. #include "internal.h"
  15. void task_mem(struct seq_file *m, struct mm_struct *mm)
  16. {
  17. unsigned long data, text, lib, swap;
  18. unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
  19. /*
  20. * Note: to minimize their overhead, mm maintains hiwater_vm and
  21. * hiwater_rss only when about to *lower* total_vm or rss. Any
  22. * collector of these hiwater stats must therefore get total_vm
  23. * and rss too, which will usually be the higher. Barriers? not
  24. * worth the effort, such snapshots can always be inconsistent.
  25. */
  26. hiwater_vm = total_vm = mm->total_vm;
  27. if (hiwater_vm < mm->hiwater_vm)
  28. hiwater_vm = mm->hiwater_vm;
  29. hiwater_rss = total_rss = get_mm_rss(mm);
  30. if (hiwater_rss < mm->hiwater_rss)
  31. hiwater_rss = mm->hiwater_rss;
  32. data = mm->total_vm - mm->shared_vm - mm->stack_vm;
  33. text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> 10;
  34. lib = (mm->exec_vm << (PAGE_SHIFT-10)) - text;
  35. swap = get_mm_counter(mm, MM_SWAPENTS);
  36. seq_printf(m,
  37. "VmPeak:\t%8lu kB\n"
  38. "VmSize:\t%8lu kB\n"
  39. "VmLck:\t%8lu kB\n"
  40. "VmHWM:\t%8lu kB\n"
  41. "VmRSS:\t%8lu kB\n"
  42. "VmData:\t%8lu kB\n"
  43. "VmStk:\t%8lu kB\n"
  44. "VmExe:\t%8lu kB\n"
  45. "VmLib:\t%8lu kB\n"
  46. "VmPTE:\t%8lu kB\n"
  47. "VmSwap:\t%8lu kB\n",
  48. hiwater_vm << (PAGE_SHIFT-10),
  49. (total_vm - mm->reserved_vm) << (PAGE_SHIFT-10),
  50. mm->locked_vm << (PAGE_SHIFT-10),
  51. hiwater_rss << (PAGE_SHIFT-10),
  52. total_rss << (PAGE_SHIFT-10),
  53. data << (PAGE_SHIFT-10),
  54. mm->stack_vm << (PAGE_SHIFT-10), text, lib,
  55. (PTRS_PER_PTE*sizeof(pte_t)*mm->nr_ptes) >> 10,
  56. swap << (PAGE_SHIFT-10));
  57. }
  58. unsigned long task_vsize(struct mm_struct *mm)
  59. {
  60. return PAGE_SIZE * mm->total_vm;
  61. }
  62. int task_statm(struct mm_struct *mm, int *shared, int *text,
  63. int *data, int *resident)
  64. {
  65. *shared = get_mm_counter(mm, MM_FILEPAGES);
  66. *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
  67. >> PAGE_SHIFT;
  68. *data = mm->total_vm - mm->shared_vm;
  69. *resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
  70. return mm->total_vm;
  71. }
  72. static void pad_len_spaces(struct seq_file *m, int len)
  73. {
  74. len = 25 + sizeof(void*) * 6 - len;
  75. if (len < 1)
  76. len = 1;
  77. seq_printf(m, "%*c", len, ' ');
  78. }
  79. static void vma_stop(struct proc_maps_private *priv, struct vm_area_struct *vma)
  80. {
  81. if (vma && vma != priv->tail_vma) {
  82. struct mm_struct *mm = vma->vm_mm;
  83. up_read(&mm->mmap_sem);
  84. mmput(mm);
  85. }
  86. }
  87. static void *m_start(struct seq_file *m, loff_t *pos)
  88. {
  89. struct proc_maps_private *priv = m->private;
  90. unsigned long last_addr = m->version;
  91. struct mm_struct *mm;
  92. struct vm_area_struct *vma, *tail_vma = NULL;
  93. loff_t l = *pos;
  94. /* Clear the per syscall fields in priv */
  95. priv->task = NULL;
  96. priv->tail_vma = NULL;
  97. /*
  98. * We remember last_addr rather than next_addr to hit with
  99. * mmap_cache most of the time. We have zero last_addr at
  100. * the beginning and also after lseek. We will have -1 last_addr
  101. * after the end of the vmas.
  102. */
  103. if (last_addr == -1UL)
  104. return NULL;
  105. priv->task = get_pid_task(priv->pid, PIDTYPE_PID);
  106. if (!priv->task)
  107. return NULL;
  108. mm = mm_for_maps(priv->task);
  109. if (!mm)
  110. return NULL;
  111. down_read(&mm->mmap_sem);
  112. tail_vma = get_gate_vma(priv->task);
  113. priv->tail_vma = tail_vma;
  114. /* Start with last addr hint */
  115. vma = find_vma(mm, last_addr);
  116. if (last_addr && vma) {
  117. vma = vma->vm_next;
  118. goto out;
  119. }
  120. /*
  121. * Check the vma index is within the range and do
  122. * sequential scan until m_index.
  123. */
  124. vma = NULL;
  125. if ((unsigned long)l < mm->map_count) {
  126. vma = mm->mmap;
  127. while (l-- && vma)
  128. vma = vma->vm_next;
  129. goto out;
  130. }
  131. if (l != mm->map_count)
  132. tail_vma = NULL; /* After gate vma */
  133. out:
  134. if (vma)
  135. return vma;
  136. /* End of vmas has been reached */
  137. m->version = (tail_vma != NULL)? 0: -1UL;
  138. up_read(&mm->mmap_sem);
  139. mmput(mm);
  140. return tail_vma;
  141. }
  142. static void *m_next(struct seq_file *m, void *v, loff_t *pos)
  143. {
  144. struct proc_maps_private *priv = m->private;
  145. struct vm_area_struct *vma = v;
  146. struct vm_area_struct *tail_vma = priv->tail_vma;
  147. (*pos)++;
  148. if (vma && (vma != tail_vma) && vma->vm_next)
  149. return vma->vm_next;
  150. vma_stop(priv, vma);
  151. return (vma != tail_vma)? tail_vma: NULL;
  152. }
  153. static void m_stop(struct seq_file *m, void *v)
  154. {
  155. struct proc_maps_private *priv = m->private;
  156. struct vm_area_struct *vma = v;
  157. vma_stop(priv, vma);
  158. if (priv->task)
  159. put_task_struct(priv->task);
  160. }
  161. static int do_maps_open(struct inode *inode, struct file *file,
  162. const struct seq_operations *ops)
  163. {
  164. struct proc_maps_private *priv;
  165. int ret = -ENOMEM;
  166. priv = kzalloc(sizeof(*priv), GFP_KERNEL);
  167. if (priv) {
  168. priv->pid = proc_pid(inode);
  169. ret = seq_open(file, ops);
  170. if (!ret) {
  171. struct seq_file *m = file->private_data;
  172. m->private = priv;
  173. } else {
  174. kfree(priv);
  175. }
  176. }
  177. return ret;
  178. }
  179. static void show_map_vma(struct seq_file *m, struct vm_area_struct *vma)
  180. {
  181. struct mm_struct *mm = vma->vm_mm;
  182. struct file *file = vma->vm_file;
  183. int flags = vma->vm_flags;
  184. unsigned long ino = 0;
  185. unsigned long long pgoff = 0;
  186. dev_t dev = 0;
  187. int len;
  188. if (file) {
  189. struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
  190. dev = inode->i_sb->s_dev;
  191. ino = inode->i_ino;
  192. pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
  193. }
  194. seq_printf(m, "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu %n",
  195. vma->vm_start,
  196. vma->vm_end,
  197. flags & VM_READ ? 'r' : '-',
  198. flags & VM_WRITE ? 'w' : '-',
  199. flags & VM_EXEC ? 'x' : '-',
  200. flags & VM_MAYSHARE ? 's' : 'p',
  201. pgoff,
  202. MAJOR(dev), MINOR(dev), ino, &len);
  203. /*
  204. * Print the dentry name for named mappings, and a
  205. * special [heap] marker for the heap:
  206. */
  207. if (file) {
  208. pad_len_spaces(m, len);
  209. seq_path(m, &file->f_path, "\n");
  210. } else {
  211. const char *name = arch_vma_name(vma);
  212. if (!name) {
  213. if (mm) {
  214. if (vma->vm_start <= mm->start_brk &&
  215. vma->vm_end >= mm->brk) {
  216. name = "[heap]";
  217. } else if (vma->vm_start <= mm->start_stack &&
  218. vma->vm_end >= mm->start_stack) {
  219. name = "[stack]";
  220. } else {
  221. unsigned long stack_start;
  222. struct proc_maps_private *pmp;
  223. pmp = m->private;
  224. stack_start = pmp->task->stack_start;
  225. if (vma->vm_start <= stack_start &&
  226. vma->vm_end >= stack_start) {
  227. pad_len_spaces(m, len);
  228. seq_printf(m,
  229. "[threadstack:%08lx]",
  230. #ifdef CONFIG_STACK_GROWSUP
  231. vma->vm_end - stack_start
  232. #else
  233. stack_start - vma->vm_start
  234. #endif
  235. );
  236. }
  237. }
  238. } else {
  239. name = "[vdso]";
  240. }
  241. }
  242. if (name) {
  243. pad_len_spaces(m, len);
  244. seq_puts(m, name);
  245. }
  246. }
  247. seq_putc(m, '\n');
  248. }
  249. static int show_map(struct seq_file *m, void *v)
  250. {
  251. struct vm_area_struct *vma = v;
  252. struct proc_maps_private *priv = m->private;
  253. struct task_struct *task = priv->task;
  254. show_map_vma(m, vma);
  255. if (m->count < m->size) /* vma is copied successfully */
  256. m->version = (vma != get_gate_vma(task))? vma->vm_start: 0;
  257. return 0;
  258. }
  259. static const struct seq_operations proc_pid_maps_op = {
  260. .start = m_start,
  261. .next = m_next,
  262. .stop = m_stop,
  263. .show = show_map
  264. };
  265. static int maps_open(struct inode *inode, struct file *file)
  266. {
  267. return do_maps_open(inode, file, &proc_pid_maps_op);
  268. }
  269. const struct file_operations proc_maps_operations = {
  270. .open = maps_open,
  271. .read = seq_read,
  272. .llseek = seq_lseek,
  273. .release = seq_release_private,
  274. };
  275. /*
  276. * Proportional Set Size(PSS): my share of RSS.
  277. *
  278. * PSS of a process is the count of pages it has in memory, where each
  279. * page is divided by the number of processes sharing it. So if a
  280. * process has 1000 pages all to itself, and 1000 shared with one other
  281. * process, its PSS will be 1500.
  282. *
  283. * To keep (accumulated) division errors low, we adopt a 64bit
  284. * fixed-point pss counter to minimize division errors. So (pss >>
  285. * PSS_SHIFT) would be the real byte count.
  286. *
  287. * A shift of 12 before division means (assuming 4K page size):
  288. * - 1M 3-user-pages add up to 8KB errors;
  289. * - supports mapcount up to 2^24, or 16M;
  290. * - supports PSS up to 2^52 bytes, or 4PB.
  291. */
  292. #define PSS_SHIFT 12
  293. #ifdef CONFIG_PROC_PAGE_MONITOR
  294. struct mem_size_stats {
  295. struct vm_area_struct *vma;
  296. unsigned long resident;
  297. unsigned long shared_clean;
  298. unsigned long shared_dirty;
  299. unsigned long private_clean;
  300. unsigned long private_dirty;
  301. unsigned long referenced;
  302. unsigned long swap;
  303. u64 pss;
  304. };
  305. static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
  306. struct mm_walk *walk)
  307. {
  308. struct mem_size_stats *mss = walk->private;
  309. struct vm_area_struct *vma = mss->vma;
  310. pte_t *pte, ptent;
  311. spinlock_t *ptl;
  312. struct page *page;
  313. int mapcount;
  314. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  315. for (; addr != end; pte++, addr += PAGE_SIZE) {
  316. ptent = *pte;
  317. if (is_swap_pte(ptent)) {
  318. mss->swap += PAGE_SIZE;
  319. continue;
  320. }
  321. if (!pte_present(ptent))
  322. continue;
  323. page = vm_normal_page(vma, addr, ptent);
  324. if (!page)
  325. continue;
  326. mss->resident += PAGE_SIZE;
  327. /* Accumulate the size in pages that have been accessed. */
  328. if (pte_young(ptent) || PageReferenced(page))
  329. mss->referenced += PAGE_SIZE;
  330. mapcount = page_mapcount(page);
  331. if (mapcount >= 2) {
  332. if (pte_dirty(ptent))
  333. mss->shared_dirty += PAGE_SIZE;
  334. else
  335. mss->shared_clean += PAGE_SIZE;
  336. mss->pss += (PAGE_SIZE << PSS_SHIFT) / mapcount;
  337. } else {
  338. if (pte_dirty(ptent))
  339. mss->private_dirty += PAGE_SIZE;
  340. else
  341. mss->private_clean += PAGE_SIZE;
  342. mss->pss += (PAGE_SIZE << PSS_SHIFT);
  343. }
  344. }
  345. pte_unmap_unlock(pte - 1, ptl);
  346. cond_resched();
  347. return 0;
  348. }
  349. static int show_smap(struct seq_file *m, void *v)
  350. {
  351. struct proc_maps_private *priv = m->private;
  352. struct task_struct *task = priv->task;
  353. struct vm_area_struct *vma = v;
  354. struct mem_size_stats mss;
  355. struct mm_walk smaps_walk = {
  356. .pmd_entry = smaps_pte_range,
  357. .mm = vma->vm_mm,
  358. .private = &mss,
  359. };
  360. memset(&mss, 0, sizeof mss);
  361. mss.vma = vma;
  362. if (vma->vm_mm && !is_vm_hugetlb_page(vma))
  363. walk_page_range(vma->vm_start, vma->vm_end, &smaps_walk);
  364. show_map_vma(m, vma);
  365. seq_printf(m,
  366. "Size: %8lu kB\n"
  367. "Rss: %8lu kB\n"
  368. "Pss: %8lu kB\n"
  369. "Shared_Clean: %8lu kB\n"
  370. "Shared_Dirty: %8lu kB\n"
  371. "Private_Clean: %8lu kB\n"
  372. "Private_Dirty: %8lu kB\n"
  373. "Referenced: %8lu kB\n"
  374. "Swap: %8lu kB\n"
  375. "KernelPageSize: %8lu kB\n"
  376. "MMUPageSize: %8lu kB\n",
  377. (vma->vm_end - vma->vm_start) >> 10,
  378. mss.resident >> 10,
  379. (unsigned long)(mss.pss >> (10 + PSS_SHIFT)),
  380. mss.shared_clean >> 10,
  381. mss.shared_dirty >> 10,
  382. mss.private_clean >> 10,
  383. mss.private_dirty >> 10,
  384. mss.referenced >> 10,
  385. mss.swap >> 10,
  386. vma_kernel_pagesize(vma) >> 10,
  387. vma_mmu_pagesize(vma) >> 10);
  388. if (m->count < m->size) /* vma is copied successfully */
  389. m->version = (vma != get_gate_vma(task)) ? vma->vm_start : 0;
  390. return 0;
  391. }
  392. static const struct seq_operations proc_pid_smaps_op = {
  393. .start = m_start,
  394. .next = m_next,
  395. .stop = m_stop,
  396. .show = show_smap
  397. };
  398. static int smaps_open(struct inode *inode, struct file *file)
  399. {
  400. return do_maps_open(inode, file, &proc_pid_smaps_op);
  401. }
  402. const struct file_operations proc_smaps_operations = {
  403. .open = smaps_open,
  404. .read = seq_read,
  405. .llseek = seq_lseek,
  406. .release = seq_release_private,
  407. };
  408. static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
  409. unsigned long end, struct mm_walk *walk)
  410. {
  411. struct vm_area_struct *vma = walk->private;
  412. pte_t *pte, ptent;
  413. spinlock_t *ptl;
  414. struct page *page;
  415. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  416. for (; addr != end; pte++, addr += PAGE_SIZE) {
  417. ptent = *pte;
  418. if (!pte_present(ptent))
  419. continue;
  420. page = vm_normal_page(vma, addr, ptent);
  421. if (!page)
  422. continue;
  423. /* Clear accessed and referenced bits. */
  424. ptep_test_and_clear_young(vma, addr, pte);
  425. ClearPageReferenced(page);
  426. }
  427. pte_unmap_unlock(pte - 1, ptl);
  428. cond_resched();
  429. return 0;
  430. }
  431. #define CLEAR_REFS_ALL 1
  432. #define CLEAR_REFS_ANON 2
  433. #define CLEAR_REFS_MAPPED 3
  434. static ssize_t clear_refs_write(struct file *file, const char __user *buf,
  435. size_t count, loff_t *ppos)
  436. {
  437. struct task_struct *task;
  438. char buffer[PROC_NUMBUF];
  439. struct mm_struct *mm;
  440. struct vm_area_struct *vma;
  441. long type;
  442. memset(buffer, 0, sizeof(buffer));
  443. if (count > sizeof(buffer) - 1)
  444. count = sizeof(buffer) - 1;
  445. if (copy_from_user(buffer, buf, count))
  446. return -EFAULT;
  447. if (strict_strtol(strstrip(buffer), 10, &type))
  448. return -EINVAL;
  449. if (type < CLEAR_REFS_ALL || type > CLEAR_REFS_MAPPED)
  450. return -EINVAL;
  451. task = get_proc_task(file->f_path.dentry->d_inode);
  452. if (!task)
  453. return -ESRCH;
  454. mm = get_task_mm(task);
  455. if (mm) {
  456. struct mm_walk clear_refs_walk = {
  457. .pmd_entry = clear_refs_pte_range,
  458. .mm = mm,
  459. };
  460. down_read(&mm->mmap_sem);
  461. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  462. clear_refs_walk.private = vma;
  463. if (is_vm_hugetlb_page(vma))
  464. continue;
  465. /*
  466. * Writing 1 to /proc/pid/clear_refs affects all pages.
  467. *
  468. * Writing 2 to /proc/pid/clear_refs only affects
  469. * Anonymous pages.
  470. *
  471. * Writing 3 to /proc/pid/clear_refs only affects file
  472. * mapped pages.
  473. */
  474. if (type == CLEAR_REFS_ANON && vma->vm_file)
  475. continue;
  476. if (type == CLEAR_REFS_MAPPED && !vma->vm_file)
  477. continue;
  478. walk_page_range(vma->vm_start, vma->vm_end,
  479. &clear_refs_walk);
  480. }
  481. flush_tlb_mm(mm);
  482. up_read(&mm->mmap_sem);
  483. mmput(mm);
  484. }
  485. put_task_struct(task);
  486. return count;
  487. }
  488. const struct file_operations proc_clear_refs_operations = {
  489. .write = clear_refs_write,
  490. };
  491. struct pagemapread {
  492. u64 __user *out, *end;
  493. };
  494. #define PM_ENTRY_BYTES sizeof(u64)
  495. #define PM_STATUS_BITS 3
  496. #define PM_STATUS_OFFSET (64 - PM_STATUS_BITS)
  497. #define PM_STATUS_MASK (((1LL << PM_STATUS_BITS) - 1) << PM_STATUS_OFFSET)
  498. #define PM_STATUS(nr) (((nr) << PM_STATUS_OFFSET) & PM_STATUS_MASK)
  499. #define PM_PSHIFT_BITS 6
  500. #define PM_PSHIFT_OFFSET (PM_STATUS_OFFSET - PM_PSHIFT_BITS)
  501. #define PM_PSHIFT_MASK (((1LL << PM_PSHIFT_BITS) - 1) << PM_PSHIFT_OFFSET)
  502. #define PM_PSHIFT(x) (((u64) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)
  503. #define PM_PFRAME_MASK ((1LL << PM_PSHIFT_OFFSET) - 1)
  504. #define PM_PFRAME(x) ((x) & PM_PFRAME_MASK)
  505. #define PM_PRESENT PM_STATUS(4LL)
  506. #define PM_SWAP PM_STATUS(2LL)
  507. #define PM_NOT_PRESENT PM_PSHIFT(PAGE_SHIFT)
  508. #define PM_END_OF_BUFFER 1
  509. static int add_to_pagemap(unsigned long addr, u64 pfn,
  510. struct pagemapread *pm)
  511. {
  512. if (put_user(pfn, pm->out))
  513. return -EFAULT;
  514. pm->out++;
  515. if (pm->out >= pm->end)
  516. return PM_END_OF_BUFFER;
  517. return 0;
  518. }
  519. static int pagemap_pte_hole(unsigned long start, unsigned long end,
  520. struct mm_walk *walk)
  521. {
  522. struct pagemapread *pm = walk->private;
  523. unsigned long addr;
  524. int err = 0;
  525. for (addr = start; addr < end; addr += PAGE_SIZE) {
  526. err = add_to_pagemap(addr, PM_NOT_PRESENT, pm);
  527. if (err)
  528. break;
  529. }
  530. return err;
  531. }
  532. static u64 swap_pte_to_pagemap_entry(pte_t pte)
  533. {
  534. swp_entry_t e = pte_to_swp_entry(pte);
  535. return swp_type(e) | (swp_offset(e) << MAX_SWAPFILES_SHIFT);
  536. }
  537. static u64 pte_to_pagemap_entry(pte_t pte)
  538. {
  539. u64 pme = 0;
  540. if (is_swap_pte(pte))
  541. pme = PM_PFRAME(swap_pte_to_pagemap_entry(pte))
  542. | PM_PSHIFT(PAGE_SHIFT) | PM_SWAP;
  543. else if (pte_present(pte))
  544. pme = PM_PFRAME(pte_pfn(pte))
  545. | PM_PSHIFT(PAGE_SHIFT) | PM_PRESENT;
  546. return pme;
  547. }
  548. static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
  549. struct mm_walk *walk)
  550. {
  551. struct vm_area_struct *vma;
  552. struct pagemapread *pm = walk->private;
  553. pte_t *pte;
  554. int err = 0;
  555. /* find the first VMA at or above 'addr' */
  556. vma = find_vma(walk->mm, addr);
  557. for (; addr != end; addr += PAGE_SIZE) {
  558. u64 pfn = PM_NOT_PRESENT;
  559. /* check to see if we've left 'vma' behind
  560. * and need a new, higher one */
  561. if (vma && (addr >= vma->vm_end))
  562. vma = find_vma(walk->mm, addr);
  563. /* check that 'vma' actually covers this address,
  564. * and that it isn't a huge page vma */
  565. if (vma && (vma->vm_start <= addr) &&
  566. !is_vm_hugetlb_page(vma)) {
  567. pte = pte_offset_map(pmd, addr);
  568. pfn = pte_to_pagemap_entry(*pte);
  569. /* unmap before userspace copy */
  570. pte_unmap(pte);
  571. }
  572. err = add_to_pagemap(addr, pfn, pm);
  573. if (err)
  574. return err;
  575. }
  576. cond_resched();
  577. return err;
  578. }
  579. static u64 huge_pte_to_pagemap_entry(pte_t pte, int offset)
  580. {
  581. u64 pme = 0;
  582. if (pte_present(pte))
  583. pme = PM_PFRAME(pte_pfn(pte) + offset)
  584. | PM_PSHIFT(PAGE_SHIFT) | PM_PRESENT;
  585. return pme;
  586. }
  587. static int pagemap_hugetlb_range(pte_t *pte, unsigned long addr,
  588. unsigned long end, struct mm_walk *walk)
  589. {
  590. struct vm_area_struct *vma;
  591. struct pagemapread *pm = walk->private;
  592. struct hstate *hs = NULL;
  593. int err = 0;
  594. vma = find_vma(walk->mm, addr);
  595. if (vma)
  596. hs = hstate_vma(vma);
  597. for (; addr != end; addr += PAGE_SIZE) {
  598. u64 pfn = PM_NOT_PRESENT;
  599. if (vma && (addr >= vma->vm_end)) {
  600. vma = find_vma(walk->mm, addr);
  601. if (vma)
  602. hs = hstate_vma(vma);
  603. }
  604. if (vma && (vma->vm_start <= addr) && is_vm_hugetlb_page(vma)) {
  605. /* calculate pfn of the "raw" page in the hugepage. */
  606. int offset = (addr & ~huge_page_mask(hs)) >> PAGE_SHIFT;
  607. pfn = huge_pte_to_pagemap_entry(*pte, offset);
  608. }
  609. err = add_to_pagemap(addr, pfn, pm);
  610. if (err)
  611. return err;
  612. }
  613. cond_resched();
  614. return err;
  615. }
  616. /*
  617. * /proc/pid/pagemap - an array mapping virtual pages to pfns
  618. *
  619. * For each page in the address space, this file contains one 64-bit entry
  620. * consisting of the following:
  621. *
  622. * Bits 0-55 page frame number (PFN) if present
  623. * Bits 0-4 swap type if swapped
  624. * Bits 5-55 swap offset if swapped
  625. * Bits 55-60 page shift (page size = 1<<page shift)
  626. * Bit 61 reserved for future use
  627. * Bit 62 page swapped
  628. * Bit 63 page present
  629. *
  630. * If the page is not present but in swap, then the PFN contains an
  631. * encoding of the swap file number and the page's offset into the
  632. * swap. Unmapped pages return a null PFN. This allows determining
  633. * precisely which pages are mapped (or in swap) and comparing mapped
  634. * pages between processes.
  635. *
  636. * Efficient users of this interface will use /proc/pid/maps to
  637. * determine which areas of memory are actually mapped and llseek to
  638. * skip over unmapped regions.
  639. */
  640. static ssize_t pagemap_read(struct file *file, char __user *buf,
  641. size_t count, loff_t *ppos)
  642. {
  643. struct task_struct *task = get_proc_task(file->f_path.dentry->d_inode);
  644. struct page **pages, *page;
  645. unsigned long uaddr, uend;
  646. struct mm_struct *mm;
  647. struct pagemapread pm;
  648. int pagecount;
  649. int ret = -ESRCH;
  650. struct mm_walk pagemap_walk = {};
  651. unsigned long src;
  652. unsigned long svpfn;
  653. unsigned long start_vaddr;
  654. unsigned long end_vaddr;
  655. if (!task)
  656. goto out;
  657. ret = -EACCES;
  658. if (!ptrace_may_access(task, PTRACE_MODE_READ))
  659. goto out_task;
  660. ret = -EINVAL;
  661. /* file position must be aligned */
  662. if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
  663. goto out_task;
  664. ret = 0;
  665. if (!count)
  666. goto out_task;
  667. mm = get_task_mm(task);
  668. if (!mm)
  669. goto out_task;
  670. uaddr = (unsigned long)buf & PAGE_MASK;
  671. uend = (unsigned long)(buf + count);
  672. pagecount = (PAGE_ALIGN(uend) - uaddr) / PAGE_SIZE;
  673. ret = 0;
  674. if (pagecount == 0)
  675. goto out_mm;
  676. pages = kcalloc(pagecount, sizeof(struct page *), GFP_KERNEL);
  677. ret = -ENOMEM;
  678. if (!pages)
  679. goto out_mm;
  680. down_read(&current->mm->mmap_sem);
  681. ret = get_user_pages(current, current->mm, uaddr, pagecount,
  682. 1, 0, pages, NULL);
  683. up_read(&current->mm->mmap_sem);
  684. if (ret < 0)
  685. goto out_free;
  686. if (ret != pagecount) {
  687. pagecount = ret;
  688. ret = -EFAULT;
  689. goto out_pages;
  690. }
  691. pm.out = (u64 __user *)buf;
  692. pm.end = (u64 __user *)(buf + count);
  693. pagemap_walk.pmd_entry = pagemap_pte_range;
  694. pagemap_walk.pte_hole = pagemap_pte_hole;
  695. pagemap_walk.hugetlb_entry = pagemap_hugetlb_range;
  696. pagemap_walk.mm = mm;
  697. pagemap_walk.private = &pm;
  698. src = *ppos;
  699. svpfn = src / PM_ENTRY_BYTES;
  700. start_vaddr = svpfn << PAGE_SHIFT;
  701. end_vaddr = TASK_SIZE_OF(task);
  702. /* watch out for wraparound */
  703. if (svpfn > TASK_SIZE_OF(task) >> PAGE_SHIFT)
  704. start_vaddr = end_vaddr;
  705. /*
  706. * The odds are that this will stop walking way
  707. * before end_vaddr, because the length of the
  708. * user buffer is tracked in "pm", and the walk
  709. * will stop when we hit the end of the buffer.
  710. */
  711. ret = walk_page_range(start_vaddr, end_vaddr, &pagemap_walk);
  712. if (ret == PM_END_OF_BUFFER)
  713. ret = 0;
  714. /* don't need mmap_sem for these, but this looks cleaner */
  715. *ppos += (char __user *)pm.out - buf;
  716. if (!ret)
  717. ret = (char __user *)pm.out - buf;
  718. out_pages:
  719. for (; pagecount; pagecount--) {
  720. page = pages[pagecount-1];
  721. if (!PageReserved(page))
  722. SetPageDirty(page);
  723. page_cache_release(page);
  724. }
  725. out_free:
  726. kfree(pages);
  727. out_mm:
  728. mmput(mm);
  729. out_task:
  730. put_task_struct(task);
  731. out:
  732. return ret;
  733. }
  734. const struct file_operations proc_pagemap_operations = {
  735. .llseek = mem_lseek, /* borrow this */
  736. .read = pagemap_read,
  737. };
  738. #endif /* CONFIG_PROC_PAGE_MONITOR */
  739. #ifdef CONFIG_NUMA
  740. extern int show_numa_map(struct seq_file *m, void *v);
  741. static const struct seq_operations proc_pid_numa_maps_op = {
  742. .start = m_start,
  743. .next = m_next,
  744. .stop = m_stop,
  745. .show = show_numa_map,
  746. };
  747. static int numa_maps_open(struct inode *inode, struct file *file)
  748. {
  749. return do_maps_open(inode, file, &proc_pid_numa_maps_op);
  750. }
  751. const struct file_operations proc_numa_maps_operations = {
  752. .open = numa_maps_open,
  753. .read = seq_read,
  754. .llseek = seq_lseek,
  755. .release = seq_release_private,
  756. };
  757. #endif