task_mmu.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767
  1. #include <linux/mm.h>
  2. #include <linux/hugetlb.h>
  3. #include <linux/mount.h>
  4. #include <linux/seq_file.h>
  5. #include <linux/highmem.h>
  6. #include <linux/ptrace.h>
  7. #include <linux/pagemap.h>
  8. #include <linux/mempolicy.h>
  9. #include <linux/swap.h>
  10. #include <linux/swapops.h>
  11. #include <asm/elf.h>
  12. #include <asm/uaccess.h>
  13. #include <asm/tlbflush.h>
  14. #include "internal.h"
  15. void task_mem(struct seq_file *m, struct mm_struct *mm)
  16. {
  17. unsigned long data, text, lib;
  18. unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
  19. /*
  20. * Note: to minimize their overhead, mm maintains hiwater_vm and
  21. * hiwater_rss only when about to *lower* total_vm or rss. Any
  22. * collector of these hiwater stats must therefore get total_vm
  23. * and rss too, which will usually be the higher. Barriers? not
  24. * worth the effort, such snapshots can always be inconsistent.
  25. */
  26. hiwater_vm = total_vm = mm->total_vm;
  27. if (hiwater_vm < mm->hiwater_vm)
  28. hiwater_vm = mm->hiwater_vm;
  29. hiwater_rss = total_rss = get_mm_rss(mm);
  30. if (hiwater_rss < mm->hiwater_rss)
  31. hiwater_rss = mm->hiwater_rss;
  32. data = mm->total_vm - mm->shared_vm - mm->stack_vm;
  33. text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> 10;
  34. lib = (mm->exec_vm << (PAGE_SHIFT-10)) - text;
  35. seq_printf(m,
  36. "VmPeak:\t%8lu kB\n"
  37. "VmSize:\t%8lu kB\n"
  38. "VmLck:\t%8lu kB\n"
  39. "VmHWM:\t%8lu kB\n"
  40. "VmRSS:\t%8lu kB\n"
  41. "VmData:\t%8lu kB\n"
  42. "VmStk:\t%8lu kB\n"
  43. "VmExe:\t%8lu kB\n"
  44. "VmLib:\t%8lu kB\n"
  45. "VmPTE:\t%8lu kB\n",
  46. hiwater_vm << (PAGE_SHIFT-10),
  47. (total_vm - mm->reserved_vm) << (PAGE_SHIFT-10),
  48. mm->locked_vm << (PAGE_SHIFT-10),
  49. hiwater_rss << (PAGE_SHIFT-10),
  50. total_rss << (PAGE_SHIFT-10),
  51. data << (PAGE_SHIFT-10),
  52. mm->stack_vm << (PAGE_SHIFT-10), text, lib,
  53. (PTRS_PER_PTE*sizeof(pte_t)*mm->nr_ptes) >> 10);
  54. }
  55. unsigned long task_vsize(struct mm_struct *mm)
  56. {
  57. return PAGE_SIZE * mm->total_vm;
  58. }
  59. int task_statm(struct mm_struct *mm, int *shared, int *text,
  60. int *data, int *resident)
  61. {
  62. *shared = get_mm_counter(mm, file_rss);
  63. *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
  64. >> PAGE_SHIFT;
  65. *data = mm->total_vm - mm->shared_vm;
  66. *resident = *shared + get_mm_counter(mm, anon_rss);
  67. return mm->total_vm;
  68. }
  69. static void pad_len_spaces(struct seq_file *m, int len)
  70. {
  71. len = 25 + sizeof(void*) * 6 - len;
  72. if (len < 1)
  73. len = 1;
  74. seq_printf(m, "%*c", len, ' ');
  75. }
  76. static void vma_stop(struct proc_maps_private *priv, struct vm_area_struct *vma)
  77. {
  78. if (vma && vma != priv->tail_vma) {
  79. struct mm_struct *mm = vma->vm_mm;
  80. up_read(&mm->mmap_sem);
  81. mmput(mm);
  82. }
  83. }
  84. static void *m_start(struct seq_file *m, loff_t *pos)
  85. {
  86. struct proc_maps_private *priv = m->private;
  87. unsigned long last_addr = m->version;
  88. struct mm_struct *mm;
  89. struct vm_area_struct *vma, *tail_vma = NULL;
  90. loff_t l = *pos;
  91. /* Clear the per syscall fields in priv */
  92. priv->task = NULL;
  93. priv->tail_vma = NULL;
  94. /*
  95. * We remember last_addr rather than next_addr to hit with
  96. * mmap_cache most of the time. We have zero last_addr at
  97. * the beginning and also after lseek. We will have -1 last_addr
  98. * after the end of the vmas.
  99. */
  100. if (last_addr == -1UL)
  101. return NULL;
  102. priv->task = get_pid_task(priv->pid, PIDTYPE_PID);
  103. if (!priv->task)
  104. return NULL;
  105. mm = mm_for_maps(priv->task);
  106. if (!mm)
  107. return NULL;
  108. tail_vma = get_gate_vma(priv->task);
  109. priv->tail_vma = tail_vma;
  110. /* Start with last addr hint */
  111. vma = find_vma(mm, last_addr);
  112. if (last_addr && vma) {
  113. vma = vma->vm_next;
  114. goto out;
  115. }
  116. /*
  117. * Check the vma index is within the range and do
  118. * sequential scan until m_index.
  119. */
  120. vma = NULL;
  121. if ((unsigned long)l < mm->map_count) {
  122. vma = mm->mmap;
  123. while (l-- && vma)
  124. vma = vma->vm_next;
  125. goto out;
  126. }
  127. if (l != mm->map_count)
  128. tail_vma = NULL; /* After gate vma */
  129. out:
  130. if (vma)
  131. return vma;
  132. /* End of vmas has been reached */
  133. m->version = (tail_vma != NULL)? 0: -1UL;
  134. up_read(&mm->mmap_sem);
  135. mmput(mm);
  136. return tail_vma;
  137. }
  138. static void *m_next(struct seq_file *m, void *v, loff_t *pos)
  139. {
  140. struct proc_maps_private *priv = m->private;
  141. struct vm_area_struct *vma = v;
  142. struct vm_area_struct *tail_vma = priv->tail_vma;
  143. (*pos)++;
  144. if (vma && (vma != tail_vma) && vma->vm_next)
  145. return vma->vm_next;
  146. vma_stop(priv, vma);
  147. return (vma != tail_vma)? tail_vma: NULL;
  148. }
  149. static void m_stop(struct seq_file *m, void *v)
  150. {
  151. struct proc_maps_private *priv = m->private;
  152. struct vm_area_struct *vma = v;
  153. vma_stop(priv, vma);
  154. if (priv->task)
  155. put_task_struct(priv->task);
  156. }
  157. static int do_maps_open(struct inode *inode, struct file *file,
  158. const struct seq_operations *ops)
  159. {
  160. struct proc_maps_private *priv;
  161. int ret = -ENOMEM;
  162. priv = kzalloc(sizeof(*priv), GFP_KERNEL);
  163. if (priv) {
  164. priv->pid = proc_pid(inode);
  165. ret = seq_open(file, ops);
  166. if (!ret) {
  167. struct seq_file *m = file->private_data;
  168. m->private = priv;
  169. } else {
  170. kfree(priv);
  171. }
  172. }
  173. return ret;
  174. }
  175. static void show_map_vma(struct seq_file *m, struct vm_area_struct *vma)
  176. {
  177. struct mm_struct *mm = vma->vm_mm;
  178. struct file *file = vma->vm_file;
  179. int flags = vma->vm_flags;
  180. unsigned long ino = 0;
  181. dev_t dev = 0;
  182. int len;
  183. if (file) {
  184. struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
  185. dev = inode->i_sb->s_dev;
  186. ino = inode->i_ino;
  187. }
  188. seq_printf(m, "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu %n",
  189. vma->vm_start,
  190. vma->vm_end,
  191. flags & VM_READ ? 'r' : '-',
  192. flags & VM_WRITE ? 'w' : '-',
  193. flags & VM_EXEC ? 'x' : '-',
  194. flags & VM_MAYSHARE ? 's' : 'p',
  195. ((loff_t)vma->vm_pgoff) << PAGE_SHIFT,
  196. MAJOR(dev), MINOR(dev), ino, &len);
  197. /*
  198. * Print the dentry name for named mappings, and a
  199. * special [heap] marker for the heap:
  200. */
  201. if (file) {
  202. pad_len_spaces(m, len);
  203. seq_path(m, &file->f_path, "\n");
  204. } else {
  205. const char *name = arch_vma_name(vma);
  206. if (!name) {
  207. if (mm) {
  208. if (vma->vm_start <= mm->start_brk &&
  209. vma->vm_end >= mm->brk) {
  210. name = "[heap]";
  211. } else if (vma->vm_start <= mm->start_stack &&
  212. vma->vm_end >= mm->start_stack) {
  213. name = "[stack]";
  214. }
  215. } else {
  216. name = "[vdso]";
  217. }
  218. }
  219. if (name) {
  220. pad_len_spaces(m, len);
  221. seq_puts(m, name);
  222. }
  223. }
  224. seq_putc(m, '\n');
  225. }
  226. static int show_map(struct seq_file *m, void *v)
  227. {
  228. struct vm_area_struct *vma = v;
  229. struct proc_maps_private *priv = m->private;
  230. struct task_struct *task = priv->task;
  231. show_map_vma(m, vma);
  232. if (m->count < m->size) /* vma is copied successfully */
  233. m->version = (vma != get_gate_vma(task))? vma->vm_start: 0;
  234. return 0;
  235. }
  236. static const struct seq_operations proc_pid_maps_op = {
  237. .start = m_start,
  238. .next = m_next,
  239. .stop = m_stop,
  240. .show = show_map
  241. };
  242. static int maps_open(struct inode *inode, struct file *file)
  243. {
  244. return do_maps_open(inode, file, &proc_pid_maps_op);
  245. }
  246. const struct file_operations proc_maps_operations = {
  247. .open = maps_open,
  248. .read = seq_read,
  249. .llseek = seq_lseek,
  250. .release = seq_release_private,
  251. };
  252. /*
  253. * Proportional Set Size(PSS): my share of RSS.
  254. *
  255. * PSS of a process is the count of pages it has in memory, where each
  256. * page is divided by the number of processes sharing it. So if a
  257. * process has 1000 pages all to itself, and 1000 shared with one other
  258. * process, its PSS will be 1500.
  259. *
  260. * To keep (accumulated) division errors low, we adopt a 64bit
  261. * fixed-point pss counter to minimize division errors. So (pss >>
  262. * PSS_SHIFT) would be the real byte count.
  263. *
  264. * A shift of 12 before division means (assuming 4K page size):
  265. * - 1M 3-user-pages add up to 8KB errors;
  266. * - supports mapcount up to 2^24, or 16M;
  267. * - supports PSS up to 2^52 bytes, or 4PB.
  268. */
  269. #define PSS_SHIFT 12
  270. #ifdef CONFIG_PROC_PAGE_MONITOR
  271. struct mem_size_stats {
  272. struct vm_area_struct *vma;
  273. unsigned long resident;
  274. unsigned long shared_clean;
  275. unsigned long shared_dirty;
  276. unsigned long private_clean;
  277. unsigned long private_dirty;
  278. unsigned long referenced;
  279. unsigned long swap;
  280. u64 pss;
  281. };
  282. static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
  283. struct mm_walk *walk)
  284. {
  285. struct mem_size_stats *mss = walk->private;
  286. struct vm_area_struct *vma = mss->vma;
  287. pte_t *pte, ptent;
  288. spinlock_t *ptl;
  289. struct page *page;
  290. int mapcount;
  291. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  292. for (; addr != end; pte++, addr += PAGE_SIZE) {
  293. ptent = *pte;
  294. if (is_swap_pte(ptent)) {
  295. mss->swap += PAGE_SIZE;
  296. continue;
  297. }
  298. if (!pte_present(ptent))
  299. continue;
  300. mss->resident += PAGE_SIZE;
  301. page = vm_normal_page(vma, addr, ptent);
  302. if (!page)
  303. continue;
  304. /* Accumulate the size in pages that have been accessed. */
  305. if (pte_young(ptent) || PageReferenced(page))
  306. mss->referenced += PAGE_SIZE;
  307. mapcount = page_mapcount(page);
  308. if (mapcount >= 2) {
  309. if (pte_dirty(ptent))
  310. mss->shared_dirty += PAGE_SIZE;
  311. else
  312. mss->shared_clean += PAGE_SIZE;
  313. mss->pss += (PAGE_SIZE << PSS_SHIFT) / mapcount;
  314. } else {
  315. if (pte_dirty(ptent))
  316. mss->private_dirty += PAGE_SIZE;
  317. else
  318. mss->private_clean += PAGE_SIZE;
  319. mss->pss += (PAGE_SIZE << PSS_SHIFT);
  320. }
  321. }
  322. pte_unmap_unlock(pte - 1, ptl);
  323. cond_resched();
  324. return 0;
  325. }
  326. static int show_smap(struct seq_file *m, void *v)
  327. {
  328. struct proc_maps_private *priv = m->private;
  329. struct task_struct *task = priv->task;
  330. struct vm_area_struct *vma = v;
  331. struct mem_size_stats mss;
  332. struct mm_walk smaps_walk = {
  333. .pmd_entry = smaps_pte_range,
  334. .mm = vma->vm_mm,
  335. .private = &mss,
  336. };
  337. memset(&mss, 0, sizeof mss);
  338. mss.vma = vma;
  339. if (vma->vm_mm && !is_vm_hugetlb_page(vma))
  340. walk_page_range(vma->vm_start, vma->vm_end, &smaps_walk);
  341. show_map_vma(m, vma);
  342. seq_printf(m,
  343. "Size: %8lu kB\n"
  344. "Rss: %8lu kB\n"
  345. "Pss: %8lu kB\n"
  346. "Shared_Clean: %8lu kB\n"
  347. "Shared_Dirty: %8lu kB\n"
  348. "Private_Clean: %8lu kB\n"
  349. "Private_Dirty: %8lu kB\n"
  350. "Referenced: %8lu kB\n"
  351. "Swap: %8lu kB\n",
  352. (vma->vm_end - vma->vm_start) >> 10,
  353. mss.resident >> 10,
  354. (unsigned long)(mss.pss >> (10 + PSS_SHIFT)),
  355. mss.shared_clean >> 10,
  356. mss.shared_dirty >> 10,
  357. mss.private_clean >> 10,
  358. mss.private_dirty >> 10,
  359. mss.referenced >> 10,
  360. mss.swap >> 10);
  361. if (m->count < m->size) /* vma is copied successfully */
  362. m->version = (vma != get_gate_vma(task)) ? vma->vm_start : 0;
  363. return 0;
  364. }
  365. static const struct seq_operations proc_pid_smaps_op = {
  366. .start = m_start,
  367. .next = m_next,
  368. .stop = m_stop,
  369. .show = show_smap
  370. };
  371. static int smaps_open(struct inode *inode, struct file *file)
  372. {
  373. return do_maps_open(inode, file, &proc_pid_smaps_op);
  374. }
  375. const struct file_operations proc_smaps_operations = {
  376. .open = smaps_open,
  377. .read = seq_read,
  378. .llseek = seq_lseek,
  379. .release = seq_release_private,
  380. };
  381. static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
  382. unsigned long end, struct mm_walk *walk)
  383. {
  384. struct vm_area_struct *vma = walk->private;
  385. pte_t *pte, ptent;
  386. spinlock_t *ptl;
  387. struct page *page;
  388. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  389. for (; addr != end; pte++, addr += PAGE_SIZE) {
  390. ptent = *pte;
  391. if (!pte_present(ptent))
  392. continue;
  393. page = vm_normal_page(vma, addr, ptent);
  394. if (!page)
  395. continue;
  396. /* Clear accessed and referenced bits. */
  397. ptep_test_and_clear_young(vma, addr, pte);
  398. ClearPageReferenced(page);
  399. }
  400. pte_unmap_unlock(pte - 1, ptl);
  401. cond_resched();
  402. return 0;
  403. }
  404. static ssize_t clear_refs_write(struct file *file, const char __user *buf,
  405. size_t count, loff_t *ppos)
  406. {
  407. struct task_struct *task;
  408. char buffer[PROC_NUMBUF], *end;
  409. struct mm_struct *mm;
  410. struct vm_area_struct *vma;
  411. memset(buffer, 0, sizeof(buffer));
  412. if (count > sizeof(buffer) - 1)
  413. count = sizeof(buffer) - 1;
  414. if (copy_from_user(buffer, buf, count))
  415. return -EFAULT;
  416. if (!simple_strtol(buffer, &end, 0))
  417. return -EINVAL;
  418. if (*end == '\n')
  419. end++;
  420. task = get_proc_task(file->f_path.dentry->d_inode);
  421. if (!task)
  422. return -ESRCH;
  423. mm = get_task_mm(task);
  424. if (mm) {
  425. struct mm_walk clear_refs_walk = {
  426. .pmd_entry = clear_refs_pte_range,
  427. .mm = mm,
  428. };
  429. down_read(&mm->mmap_sem);
  430. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  431. clear_refs_walk.private = vma;
  432. if (!is_vm_hugetlb_page(vma))
  433. walk_page_range(vma->vm_start, vma->vm_end,
  434. &clear_refs_walk);
  435. }
  436. flush_tlb_mm(mm);
  437. up_read(&mm->mmap_sem);
  438. mmput(mm);
  439. }
  440. put_task_struct(task);
  441. if (end - buffer == 0)
  442. return -EIO;
  443. return end - buffer;
  444. }
  445. const struct file_operations proc_clear_refs_operations = {
  446. .write = clear_refs_write,
  447. };
  448. struct pagemapread {
  449. u64 __user *out, *end;
  450. };
  451. #define PM_ENTRY_BYTES sizeof(u64)
  452. #define PM_STATUS_BITS 3
  453. #define PM_STATUS_OFFSET (64 - PM_STATUS_BITS)
  454. #define PM_STATUS_MASK (((1LL << PM_STATUS_BITS) - 1) << PM_STATUS_OFFSET)
  455. #define PM_STATUS(nr) (((nr) << PM_STATUS_OFFSET) & PM_STATUS_MASK)
  456. #define PM_PSHIFT_BITS 6
  457. #define PM_PSHIFT_OFFSET (PM_STATUS_OFFSET - PM_PSHIFT_BITS)
  458. #define PM_PSHIFT_MASK (((1LL << PM_PSHIFT_BITS) - 1) << PM_PSHIFT_OFFSET)
  459. #define PM_PSHIFT(x) (((u64) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)
  460. #define PM_PFRAME_MASK ((1LL << PM_PSHIFT_OFFSET) - 1)
  461. #define PM_PFRAME(x) ((x) & PM_PFRAME_MASK)
  462. #define PM_PRESENT PM_STATUS(4LL)
  463. #define PM_SWAP PM_STATUS(2LL)
  464. #define PM_NOT_PRESENT PM_PSHIFT(PAGE_SHIFT)
  465. #define PM_END_OF_BUFFER 1
  466. static int add_to_pagemap(unsigned long addr, u64 pfn,
  467. struct pagemapread *pm)
  468. {
  469. if (put_user(pfn, pm->out))
  470. return -EFAULT;
  471. pm->out++;
  472. if (pm->out >= pm->end)
  473. return PM_END_OF_BUFFER;
  474. return 0;
  475. }
  476. static int pagemap_pte_hole(unsigned long start, unsigned long end,
  477. struct mm_walk *walk)
  478. {
  479. struct pagemapread *pm = walk->private;
  480. unsigned long addr;
  481. int err = 0;
  482. for (addr = start; addr < end; addr += PAGE_SIZE) {
  483. err = add_to_pagemap(addr, PM_NOT_PRESENT, pm);
  484. if (err)
  485. break;
  486. }
  487. return err;
  488. }
  489. static u64 swap_pte_to_pagemap_entry(pte_t pte)
  490. {
  491. swp_entry_t e = pte_to_swp_entry(pte);
  492. return swp_type(e) | (swp_offset(e) << MAX_SWAPFILES_SHIFT);
  493. }
  494. static unsigned long pte_to_pagemap_entry(pte_t pte)
  495. {
  496. unsigned long pme = 0;
  497. if (is_swap_pte(pte))
  498. pme = PM_PFRAME(swap_pte_to_pagemap_entry(pte))
  499. | PM_PSHIFT(PAGE_SHIFT) | PM_SWAP;
  500. else if (pte_present(pte))
  501. pme = PM_PFRAME(pte_pfn(pte))
  502. | PM_PSHIFT(PAGE_SHIFT) | PM_PRESENT;
  503. return pme;
  504. }
  505. static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
  506. struct mm_walk *walk)
  507. {
  508. struct vm_area_struct *vma;
  509. struct pagemapread *pm = walk->private;
  510. pte_t *pte;
  511. int err = 0;
  512. /* find the first VMA at or above 'addr' */
  513. vma = find_vma(walk->mm, addr);
  514. for (; addr != end; addr += PAGE_SIZE) {
  515. u64 pfn = PM_NOT_PRESENT;
  516. /* check to see if we've left 'vma' behind
  517. * and need a new, higher one */
  518. if (vma && (addr >= vma->vm_end))
  519. vma = find_vma(walk->mm, addr);
  520. /* check that 'vma' actually covers this address,
  521. * and that it isn't a huge page vma */
  522. if (vma && (vma->vm_start <= addr) &&
  523. !is_vm_hugetlb_page(vma)) {
  524. pte = pte_offset_map(pmd, addr);
  525. pfn = pte_to_pagemap_entry(*pte);
  526. /* unmap before userspace copy */
  527. pte_unmap(pte);
  528. }
  529. err = add_to_pagemap(addr, pfn, pm);
  530. if (err)
  531. return err;
  532. }
  533. cond_resched();
  534. return err;
  535. }
  536. /*
  537. * /proc/pid/pagemap - an array mapping virtual pages to pfns
  538. *
  539. * For each page in the address space, this file contains one 64-bit entry
  540. * consisting of the following:
  541. *
  542. * Bits 0-55 page frame number (PFN) if present
  543. * Bits 0-4 swap type if swapped
  544. * Bits 5-55 swap offset if swapped
  545. * Bits 55-60 page shift (page size = 1<<page shift)
  546. * Bit 61 reserved for future use
  547. * Bit 62 page swapped
  548. * Bit 63 page present
  549. *
  550. * If the page is not present but in swap, then the PFN contains an
  551. * encoding of the swap file number and the page's offset into the
  552. * swap. Unmapped pages return a null PFN. This allows determining
  553. * precisely which pages are mapped (or in swap) and comparing mapped
  554. * pages between processes.
  555. *
  556. * Efficient users of this interface will use /proc/pid/maps to
  557. * determine which areas of memory are actually mapped and llseek to
  558. * skip over unmapped regions.
  559. */
  560. static ssize_t pagemap_read(struct file *file, char __user *buf,
  561. size_t count, loff_t *ppos)
  562. {
  563. struct task_struct *task = get_proc_task(file->f_path.dentry->d_inode);
  564. struct page **pages, *page;
  565. unsigned long uaddr, uend;
  566. struct mm_struct *mm;
  567. struct pagemapread pm;
  568. int pagecount;
  569. int ret = -ESRCH;
  570. struct mm_walk pagemap_walk = {};
  571. unsigned long src;
  572. unsigned long svpfn;
  573. unsigned long start_vaddr;
  574. unsigned long end_vaddr;
  575. if (!task)
  576. goto out;
  577. ret = -EACCES;
  578. if (!ptrace_may_access(task, PTRACE_MODE_READ))
  579. goto out_task;
  580. ret = -EINVAL;
  581. /* file position must be aligned */
  582. if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
  583. goto out_task;
  584. ret = 0;
  585. mm = get_task_mm(task);
  586. if (!mm)
  587. goto out_task;
  588. uaddr = (unsigned long)buf & PAGE_MASK;
  589. uend = (unsigned long)(buf + count);
  590. pagecount = (PAGE_ALIGN(uend) - uaddr) / PAGE_SIZE;
  591. ret = 0;
  592. if (pagecount == 0)
  593. goto out_mm;
  594. pages = kcalloc(pagecount, sizeof(struct page *), GFP_KERNEL);
  595. ret = -ENOMEM;
  596. if (!pages)
  597. goto out_mm;
  598. down_read(&current->mm->mmap_sem);
  599. ret = get_user_pages(current, current->mm, uaddr, pagecount,
  600. 1, 0, pages, NULL);
  601. up_read(&current->mm->mmap_sem);
  602. if (ret < 0)
  603. goto out_free;
  604. if (ret != pagecount) {
  605. pagecount = ret;
  606. ret = -EFAULT;
  607. goto out_pages;
  608. }
  609. pm.out = (u64 *)buf;
  610. pm.end = (u64 *)(buf + count);
  611. pagemap_walk.pmd_entry = pagemap_pte_range;
  612. pagemap_walk.pte_hole = pagemap_pte_hole;
  613. pagemap_walk.mm = mm;
  614. pagemap_walk.private = &pm;
  615. src = *ppos;
  616. svpfn = src / PM_ENTRY_BYTES;
  617. start_vaddr = svpfn << PAGE_SHIFT;
  618. end_vaddr = TASK_SIZE_OF(task);
  619. /* watch out for wraparound */
  620. if (svpfn > TASK_SIZE_OF(task) >> PAGE_SHIFT)
  621. start_vaddr = end_vaddr;
  622. /*
  623. * The odds are that this will stop walking way
  624. * before end_vaddr, because the length of the
  625. * user buffer is tracked in "pm", and the walk
  626. * will stop when we hit the end of the buffer.
  627. */
  628. ret = walk_page_range(start_vaddr, end_vaddr, &pagemap_walk);
  629. if (ret == PM_END_OF_BUFFER)
  630. ret = 0;
  631. /* don't need mmap_sem for these, but this looks cleaner */
  632. *ppos += (char *)pm.out - buf;
  633. if (!ret)
  634. ret = (char *)pm.out - buf;
  635. out_pages:
  636. for (; pagecount; pagecount--) {
  637. page = pages[pagecount-1];
  638. if (!PageReserved(page))
  639. SetPageDirty(page);
  640. page_cache_release(page);
  641. }
  642. out_free:
  643. kfree(pages);
  644. out_mm:
  645. mmput(mm);
  646. out_task:
  647. put_task_struct(task);
  648. out:
  649. return ret;
  650. }
  651. const struct file_operations proc_pagemap_operations = {
  652. .llseek = mem_lseek, /* borrow this */
  653. .read = pagemap_read,
  654. };
  655. #endif /* CONFIG_PROC_PAGE_MONITOR */
  656. #ifdef CONFIG_NUMA
  657. extern int show_numa_map(struct seq_file *m, void *v);
  658. static const struct seq_operations proc_pid_numa_maps_op = {
  659. .start = m_start,
  660. .next = m_next,
  661. .stop = m_stop,
  662. .show = show_numa_map,
  663. };
  664. static int numa_maps_open(struct inode *inode, struct file *file)
  665. {
  666. return do_maps_open(inode, file, &proc_pid_numa_maps_op);
  667. }
  668. const struct file_operations proc_numa_maps_operations = {
  669. .open = numa_maps_open,
  670. .read = seq_read,
  671. .llseek = seq_lseek,
  672. .release = seq_release_private,
  673. };
  674. #endif