inode.c 24 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036
  1. /*
  2. * hugetlbpage-backed filesystem. Based on ramfs.
  3. *
  4. * William Irwin, 2002
  5. *
  6. * Copyright (C) 2002 Linus Torvalds.
  7. */
  8. #include <linux/module.h>
  9. #include <linux/thread_info.h>
  10. #include <asm/current.h>
  11. #include <linux/sched.h> /* remove ASAP */
  12. #include <linux/fs.h>
  13. #include <linux/mount.h>
  14. #include <linux/file.h>
  15. #include <linux/kernel.h>
  16. #include <linux/writeback.h>
  17. #include <linux/pagemap.h>
  18. #include <linux/highmem.h>
  19. #include <linux/init.h>
  20. #include <linux/string.h>
  21. #include <linux/capability.h>
  22. #include <linux/ctype.h>
  23. #include <linux/backing-dev.h>
  24. #include <linux/hugetlb.h>
  25. #include <linux/pagevec.h>
  26. #include <linux/parser.h>
  27. #include <linux/mman.h>
  28. #include <linux/slab.h>
  29. #include <linux/dnotify.h>
  30. #include <linux/statfs.h>
  31. #include <linux/security.h>
  32. #include <linux/ima.h>
  33. #include <linux/magic.h>
  34. #include <asm/uaccess.h>
  35. static const struct super_operations hugetlbfs_ops;
  36. static const struct address_space_operations hugetlbfs_aops;
  37. const struct file_operations hugetlbfs_file_operations;
  38. static const struct inode_operations hugetlbfs_dir_inode_operations;
  39. static const struct inode_operations hugetlbfs_inode_operations;
  40. static struct backing_dev_info hugetlbfs_backing_dev_info = {
  41. .name = "hugetlbfs",
  42. .ra_pages = 0, /* No readahead */
  43. .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK,
  44. };
  45. int sysctl_hugetlb_shm_group;
  46. enum {
  47. Opt_size, Opt_nr_inodes,
  48. Opt_mode, Opt_uid, Opt_gid,
  49. Opt_pagesize,
  50. Opt_err,
  51. };
  52. static const match_table_t tokens = {
  53. {Opt_size, "size=%s"},
  54. {Opt_nr_inodes, "nr_inodes=%s"},
  55. {Opt_mode, "mode=%o"},
  56. {Opt_uid, "uid=%u"},
  57. {Opt_gid, "gid=%u"},
  58. {Opt_pagesize, "pagesize=%s"},
  59. {Opt_err, NULL},
  60. };
  61. static void huge_pagevec_release(struct pagevec *pvec)
  62. {
  63. int i;
  64. for (i = 0; i < pagevec_count(pvec); ++i)
  65. put_page(pvec->pages[i]);
  66. pagevec_reinit(pvec);
  67. }
  68. static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
  69. {
  70. struct inode *inode = file->f_path.dentry->d_inode;
  71. loff_t len, vma_len;
  72. int ret;
  73. struct hstate *h = hstate_file(file);
  74. /*
  75. * vma address alignment (but not the pgoff alignment) has
  76. * already been checked by prepare_hugepage_range. If you add
  77. * any error returns here, do so after setting VM_HUGETLB, so
  78. * is_vm_hugetlb_page tests below unmap_region go the right
  79. * way when do_mmap_pgoff unwinds (may be important on powerpc
  80. * and ia64).
  81. */
  82. vma->vm_flags |= VM_HUGETLB | VM_RESERVED;
  83. vma->vm_ops = &hugetlb_vm_ops;
  84. if (vma->vm_pgoff & ~(huge_page_mask(h) >> PAGE_SHIFT))
  85. return -EINVAL;
  86. vma_len = (loff_t)(vma->vm_end - vma->vm_start);
  87. mutex_lock(&inode->i_mutex);
  88. file_accessed(file);
  89. ret = -ENOMEM;
  90. len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
  91. if (hugetlb_reserve_pages(inode,
  92. vma->vm_pgoff >> huge_page_order(h),
  93. len >> huge_page_shift(h), vma,
  94. vma->vm_flags))
  95. goto out;
  96. ret = 0;
  97. hugetlb_prefault_arch_hook(vma->vm_mm);
  98. if (vma->vm_flags & VM_WRITE && inode->i_size < len)
  99. inode->i_size = len;
  100. out:
  101. mutex_unlock(&inode->i_mutex);
  102. return ret;
  103. }
  104. /*
  105. * Called under down_write(mmap_sem).
  106. */
  107. #ifndef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
  108. static unsigned long
  109. hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
  110. unsigned long len, unsigned long pgoff, unsigned long flags)
  111. {
  112. struct mm_struct *mm = current->mm;
  113. struct vm_area_struct *vma;
  114. unsigned long start_addr;
  115. struct hstate *h = hstate_file(file);
  116. if (len & ~huge_page_mask(h))
  117. return -EINVAL;
  118. if (len > TASK_SIZE)
  119. return -ENOMEM;
  120. if (flags & MAP_FIXED) {
  121. if (prepare_hugepage_range(file, addr, len))
  122. return -EINVAL;
  123. return addr;
  124. }
  125. if (addr) {
  126. addr = ALIGN(addr, huge_page_size(h));
  127. vma = find_vma(mm, addr);
  128. if (TASK_SIZE - len >= addr &&
  129. (!vma || addr + len <= vma->vm_start))
  130. return addr;
  131. }
  132. start_addr = mm->free_area_cache;
  133. if (len <= mm->cached_hole_size)
  134. start_addr = TASK_UNMAPPED_BASE;
  135. full_search:
  136. addr = ALIGN(start_addr, huge_page_size(h));
  137. for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
  138. /* At this point: (!vma || addr < vma->vm_end). */
  139. if (TASK_SIZE - len < addr) {
  140. /*
  141. * Start a new search - just in case we missed
  142. * some holes.
  143. */
  144. if (start_addr != TASK_UNMAPPED_BASE) {
  145. start_addr = TASK_UNMAPPED_BASE;
  146. goto full_search;
  147. }
  148. return -ENOMEM;
  149. }
  150. if (!vma || addr + len <= vma->vm_start)
  151. return addr;
  152. addr = ALIGN(vma->vm_end, huge_page_size(h));
  153. }
  154. }
  155. #endif
  156. static int
  157. hugetlbfs_read_actor(struct page *page, unsigned long offset,
  158. char __user *buf, unsigned long count,
  159. unsigned long size)
  160. {
  161. char *kaddr;
  162. unsigned long left, copied = 0;
  163. int i, chunksize;
  164. if (size > count)
  165. size = count;
  166. /* Find which 4k chunk and offset with in that chunk */
  167. i = offset >> PAGE_CACHE_SHIFT;
  168. offset = offset & ~PAGE_CACHE_MASK;
  169. while (size) {
  170. chunksize = PAGE_CACHE_SIZE;
  171. if (offset)
  172. chunksize -= offset;
  173. if (chunksize > size)
  174. chunksize = size;
  175. kaddr = kmap(&page[i]);
  176. left = __copy_to_user(buf, kaddr + offset, chunksize);
  177. kunmap(&page[i]);
  178. if (left) {
  179. copied += (chunksize - left);
  180. break;
  181. }
  182. offset = 0;
  183. size -= chunksize;
  184. buf += chunksize;
  185. copied += chunksize;
  186. i++;
  187. }
  188. return copied ? copied : -EFAULT;
  189. }
  190. /*
  191. * Support for read() - Find the page attached to f_mapping and copy out the
  192. * data. Its *very* similar to do_generic_mapping_read(), we can't use that
  193. * since it has PAGE_CACHE_SIZE assumptions.
  194. */
  195. static ssize_t hugetlbfs_read(struct file *filp, char __user *buf,
  196. size_t len, loff_t *ppos)
  197. {
  198. struct hstate *h = hstate_file(filp);
  199. struct address_space *mapping = filp->f_mapping;
  200. struct inode *inode = mapping->host;
  201. unsigned long index = *ppos >> huge_page_shift(h);
  202. unsigned long offset = *ppos & ~huge_page_mask(h);
  203. unsigned long end_index;
  204. loff_t isize;
  205. ssize_t retval = 0;
  206. mutex_lock(&inode->i_mutex);
  207. /* validate length */
  208. if (len == 0)
  209. goto out;
  210. isize = i_size_read(inode);
  211. if (!isize)
  212. goto out;
  213. end_index = (isize - 1) >> huge_page_shift(h);
  214. for (;;) {
  215. struct page *page;
  216. unsigned long nr, ret;
  217. int ra;
  218. /* nr is the maximum number of bytes to copy from this page */
  219. nr = huge_page_size(h);
  220. if (index >= end_index) {
  221. if (index > end_index)
  222. goto out;
  223. nr = ((isize - 1) & ~huge_page_mask(h)) + 1;
  224. if (nr <= offset) {
  225. goto out;
  226. }
  227. }
  228. nr = nr - offset;
  229. /* Find the page */
  230. page = find_get_page(mapping, index);
  231. if (unlikely(page == NULL)) {
  232. /*
  233. * We have a HOLE, zero out the user-buffer for the
  234. * length of the hole or request.
  235. */
  236. ret = len < nr ? len : nr;
  237. if (clear_user(buf, ret))
  238. ra = -EFAULT;
  239. else
  240. ra = 0;
  241. } else {
  242. /*
  243. * We have the page, copy it to user space buffer.
  244. */
  245. ra = hugetlbfs_read_actor(page, offset, buf, len, nr);
  246. ret = ra;
  247. }
  248. if (ra < 0) {
  249. if (retval == 0)
  250. retval = ra;
  251. if (page)
  252. page_cache_release(page);
  253. goto out;
  254. }
  255. offset += ret;
  256. retval += ret;
  257. len -= ret;
  258. index += offset >> huge_page_shift(h);
  259. offset &= ~huge_page_mask(h);
  260. if (page)
  261. page_cache_release(page);
  262. /* short read or no more work */
  263. if ((ret != nr) || (len == 0))
  264. break;
  265. }
  266. out:
  267. *ppos = ((loff_t)index << huge_page_shift(h)) + offset;
  268. mutex_unlock(&inode->i_mutex);
  269. return retval;
  270. }
  271. static int hugetlbfs_write_begin(struct file *file,
  272. struct address_space *mapping,
  273. loff_t pos, unsigned len, unsigned flags,
  274. struct page **pagep, void **fsdata)
  275. {
  276. return -EINVAL;
  277. }
  278. static int hugetlbfs_write_end(struct file *file, struct address_space *mapping,
  279. loff_t pos, unsigned len, unsigned copied,
  280. struct page *page, void *fsdata)
  281. {
  282. BUG();
  283. return -EINVAL;
  284. }
  285. static void truncate_huge_page(struct page *page)
  286. {
  287. cancel_dirty_page(page, /* No IO accounting for huge pages? */0);
  288. ClearPageUptodate(page);
  289. remove_from_page_cache(page);
  290. put_page(page);
  291. }
  292. static void truncate_hugepages(struct inode *inode, loff_t lstart)
  293. {
  294. struct hstate *h = hstate_inode(inode);
  295. struct address_space *mapping = &inode->i_data;
  296. const pgoff_t start = lstart >> huge_page_shift(h);
  297. struct pagevec pvec;
  298. pgoff_t next;
  299. int i, freed = 0;
  300. pagevec_init(&pvec, 0);
  301. next = start;
  302. while (1) {
  303. if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
  304. if (next == start)
  305. break;
  306. next = start;
  307. continue;
  308. }
  309. for (i = 0; i < pagevec_count(&pvec); ++i) {
  310. struct page *page = pvec.pages[i];
  311. lock_page(page);
  312. if (page->index > next)
  313. next = page->index;
  314. ++next;
  315. truncate_huge_page(page);
  316. unlock_page(page);
  317. freed++;
  318. }
  319. huge_pagevec_release(&pvec);
  320. }
  321. BUG_ON(!lstart && mapping->nrpages);
  322. hugetlb_unreserve_pages(inode, start, freed);
  323. }
  324. static void hugetlbfs_delete_inode(struct inode *inode)
  325. {
  326. truncate_hugepages(inode, 0);
  327. clear_inode(inode);
  328. }
  329. static void hugetlbfs_forget_inode(struct inode *inode) __releases(inode_lock)
  330. {
  331. if (generic_detach_inode(inode)) {
  332. truncate_hugepages(inode, 0);
  333. clear_inode(inode);
  334. destroy_inode(inode);
  335. }
  336. }
  337. static void hugetlbfs_drop_inode(struct inode *inode)
  338. {
  339. if (!inode->i_nlink)
  340. generic_delete_inode(inode);
  341. else
  342. hugetlbfs_forget_inode(inode);
  343. }
  344. static inline void
  345. hugetlb_vmtruncate_list(struct prio_tree_root *root, pgoff_t pgoff)
  346. {
  347. struct vm_area_struct *vma;
  348. struct prio_tree_iter iter;
  349. vma_prio_tree_foreach(vma, &iter, root, pgoff, ULONG_MAX) {
  350. unsigned long v_offset;
  351. /*
  352. * Can the expression below overflow on 32-bit arches?
  353. * No, because the prio_tree returns us only those vmas
  354. * which overlap the truncated area starting at pgoff,
  355. * and no vma on a 32-bit arch can span beyond the 4GB.
  356. */
  357. if (vma->vm_pgoff < pgoff)
  358. v_offset = (pgoff - vma->vm_pgoff) << PAGE_SHIFT;
  359. else
  360. v_offset = 0;
  361. __unmap_hugepage_range(vma,
  362. vma->vm_start + v_offset, vma->vm_end, NULL);
  363. }
  364. }
  365. static int hugetlb_vmtruncate(struct inode *inode, loff_t offset)
  366. {
  367. pgoff_t pgoff;
  368. struct address_space *mapping = inode->i_mapping;
  369. struct hstate *h = hstate_inode(inode);
  370. BUG_ON(offset & ~huge_page_mask(h));
  371. pgoff = offset >> PAGE_SHIFT;
  372. i_size_write(inode, offset);
  373. spin_lock(&mapping->i_mmap_lock);
  374. if (!prio_tree_empty(&mapping->i_mmap))
  375. hugetlb_vmtruncate_list(&mapping->i_mmap, pgoff);
  376. spin_unlock(&mapping->i_mmap_lock);
  377. truncate_hugepages(inode, offset);
  378. return 0;
  379. }
  380. static int hugetlbfs_setattr(struct dentry *dentry, struct iattr *attr)
  381. {
  382. struct inode *inode = dentry->d_inode;
  383. struct hstate *h = hstate_inode(inode);
  384. int error;
  385. unsigned int ia_valid = attr->ia_valid;
  386. BUG_ON(!inode);
  387. error = inode_change_ok(inode, attr);
  388. if (error)
  389. goto out;
  390. if (ia_valid & ATTR_SIZE) {
  391. error = -EINVAL;
  392. if (!(attr->ia_size & ~huge_page_mask(h)))
  393. error = hugetlb_vmtruncate(inode, attr->ia_size);
  394. if (error)
  395. goto out;
  396. attr->ia_valid &= ~ATTR_SIZE;
  397. }
  398. error = inode_setattr(inode, attr);
  399. out:
  400. return error;
  401. }
  402. static struct inode *hugetlbfs_get_inode(struct super_block *sb, uid_t uid,
  403. gid_t gid, int mode, dev_t dev)
  404. {
  405. struct inode *inode;
  406. inode = new_inode(sb);
  407. if (inode) {
  408. struct hugetlbfs_inode_info *info;
  409. inode->i_mode = mode;
  410. inode->i_uid = uid;
  411. inode->i_gid = gid;
  412. inode->i_mapping->a_ops = &hugetlbfs_aops;
  413. inode->i_mapping->backing_dev_info =&hugetlbfs_backing_dev_info;
  414. inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
  415. INIT_LIST_HEAD(&inode->i_mapping->private_list);
  416. info = HUGETLBFS_I(inode);
  417. /*
  418. * The policy is initialized here even if we are creating a
  419. * private inode because initialization simply creates an
  420. * an empty rb tree and calls spin_lock_init(), later when we
  421. * call mpol_free_shared_policy() it will just return because
  422. * the rb tree will still be empty.
  423. */
  424. mpol_shared_policy_init(&info->policy, NULL);
  425. switch (mode & S_IFMT) {
  426. default:
  427. init_special_inode(inode, mode, dev);
  428. break;
  429. case S_IFREG:
  430. inode->i_op = &hugetlbfs_inode_operations;
  431. inode->i_fop = &hugetlbfs_file_operations;
  432. break;
  433. case S_IFDIR:
  434. inode->i_op = &hugetlbfs_dir_inode_operations;
  435. inode->i_fop = &simple_dir_operations;
  436. /* directory inodes start off with i_nlink == 2 (for "." entry) */
  437. inc_nlink(inode);
  438. break;
  439. case S_IFLNK:
  440. inode->i_op = &page_symlink_inode_operations;
  441. break;
  442. }
  443. }
  444. return inode;
  445. }
  446. /*
  447. * File creation. Allocate an inode, and we're done..
  448. */
  449. static int hugetlbfs_mknod(struct inode *dir,
  450. struct dentry *dentry, int mode, dev_t dev)
  451. {
  452. struct inode *inode;
  453. int error = -ENOSPC;
  454. gid_t gid;
  455. if (dir->i_mode & S_ISGID) {
  456. gid = dir->i_gid;
  457. if (S_ISDIR(mode))
  458. mode |= S_ISGID;
  459. } else {
  460. gid = current_fsgid();
  461. }
  462. inode = hugetlbfs_get_inode(dir->i_sb, current_fsuid(), gid, mode, dev);
  463. if (inode) {
  464. dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  465. d_instantiate(dentry, inode);
  466. dget(dentry); /* Extra count - pin the dentry in core */
  467. error = 0;
  468. }
  469. return error;
  470. }
  471. static int hugetlbfs_mkdir(struct inode *dir, struct dentry *dentry, int mode)
  472. {
  473. int retval = hugetlbfs_mknod(dir, dentry, mode | S_IFDIR, 0);
  474. if (!retval)
  475. inc_nlink(dir);
  476. return retval;
  477. }
  478. static int hugetlbfs_create(struct inode *dir, struct dentry *dentry, int mode, struct nameidata *nd)
  479. {
  480. return hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0);
  481. }
  482. static int hugetlbfs_symlink(struct inode *dir,
  483. struct dentry *dentry, const char *symname)
  484. {
  485. struct inode *inode;
  486. int error = -ENOSPC;
  487. gid_t gid;
  488. if (dir->i_mode & S_ISGID)
  489. gid = dir->i_gid;
  490. else
  491. gid = current_fsgid();
  492. inode = hugetlbfs_get_inode(dir->i_sb, current_fsuid(),
  493. gid, S_IFLNK|S_IRWXUGO, 0);
  494. if (inode) {
  495. int l = strlen(symname)+1;
  496. error = page_symlink(inode, symname, l);
  497. if (!error) {
  498. d_instantiate(dentry, inode);
  499. dget(dentry);
  500. } else
  501. iput(inode);
  502. }
  503. dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  504. return error;
  505. }
  506. /*
  507. * mark the head page dirty
  508. */
  509. static int hugetlbfs_set_page_dirty(struct page *page)
  510. {
  511. struct page *head = compound_head(page);
  512. SetPageDirty(head);
  513. return 0;
  514. }
  515. static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
  516. {
  517. struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
  518. struct hstate *h = hstate_inode(dentry->d_inode);
  519. buf->f_type = HUGETLBFS_MAGIC;
  520. buf->f_bsize = huge_page_size(h);
  521. if (sbinfo) {
  522. spin_lock(&sbinfo->stat_lock);
  523. /* If no limits set, just report 0 for max/free/used
  524. * blocks, like simple_statfs() */
  525. if (sbinfo->max_blocks >= 0) {
  526. buf->f_blocks = sbinfo->max_blocks;
  527. buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
  528. buf->f_files = sbinfo->max_inodes;
  529. buf->f_ffree = sbinfo->free_inodes;
  530. }
  531. spin_unlock(&sbinfo->stat_lock);
  532. }
  533. buf->f_namelen = NAME_MAX;
  534. return 0;
  535. }
  536. static void hugetlbfs_put_super(struct super_block *sb)
  537. {
  538. struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);
  539. if (sbi) {
  540. sb->s_fs_info = NULL;
  541. kfree(sbi);
  542. }
  543. }
  544. static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
  545. {
  546. if (sbinfo->free_inodes >= 0) {
  547. spin_lock(&sbinfo->stat_lock);
  548. if (unlikely(!sbinfo->free_inodes)) {
  549. spin_unlock(&sbinfo->stat_lock);
  550. return 0;
  551. }
  552. sbinfo->free_inodes--;
  553. spin_unlock(&sbinfo->stat_lock);
  554. }
  555. return 1;
  556. }
  557. static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
  558. {
  559. if (sbinfo->free_inodes >= 0) {
  560. spin_lock(&sbinfo->stat_lock);
  561. sbinfo->free_inodes++;
  562. spin_unlock(&sbinfo->stat_lock);
  563. }
  564. }
  565. static struct kmem_cache *hugetlbfs_inode_cachep;
  566. static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
  567. {
  568. struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
  569. struct hugetlbfs_inode_info *p;
  570. if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
  571. return NULL;
  572. p = kmem_cache_alloc(hugetlbfs_inode_cachep, GFP_KERNEL);
  573. if (unlikely(!p)) {
  574. hugetlbfs_inc_free_inodes(sbinfo);
  575. return NULL;
  576. }
  577. return &p->vfs_inode;
  578. }
  579. static void hugetlbfs_destroy_inode(struct inode *inode)
  580. {
  581. hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
  582. mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);
  583. kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
  584. }
  585. static const struct address_space_operations hugetlbfs_aops = {
  586. .write_begin = hugetlbfs_write_begin,
  587. .write_end = hugetlbfs_write_end,
  588. .set_page_dirty = hugetlbfs_set_page_dirty,
  589. };
  590. static void init_once(void *foo)
  591. {
  592. struct hugetlbfs_inode_info *ei = (struct hugetlbfs_inode_info *)foo;
  593. inode_init_once(&ei->vfs_inode);
  594. }
  595. const struct file_operations hugetlbfs_file_operations = {
  596. .read = hugetlbfs_read,
  597. .mmap = hugetlbfs_file_mmap,
  598. .fsync = simple_sync_file,
  599. .get_unmapped_area = hugetlb_get_unmapped_area,
  600. };
  601. static const struct inode_operations hugetlbfs_dir_inode_operations = {
  602. .create = hugetlbfs_create,
  603. .lookup = simple_lookup,
  604. .link = simple_link,
  605. .unlink = simple_unlink,
  606. .symlink = hugetlbfs_symlink,
  607. .mkdir = hugetlbfs_mkdir,
  608. .rmdir = simple_rmdir,
  609. .mknod = hugetlbfs_mknod,
  610. .rename = simple_rename,
  611. .setattr = hugetlbfs_setattr,
  612. };
  613. static const struct inode_operations hugetlbfs_inode_operations = {
  614. .setattr = hugetlbfs_setattr,
  615. };
  616. static const struct super_operations hugetlbfs_ops = {
  617. .alloc_inode = hugetlbfs_alloc_inode,
  618. .destroy_inode = hugetlbfs_destroy_inode,
  619. .statfs = hugetlbfs_statfs,
  620. .delete_inode = hugetlbfs_delete_inode,
  621. .drop_inode = hugetlbfs_drop_inode,
  622. .put_super = hugetlbfs_put_super,
  623. .show_options = generic_show_options,
  624. };
  625. static int
  626. hugetlbfs_parse_options(char *options, struct hugetlbfs_config *pconfig)
  627. {
  628. char *p, *rest;
  629. substring_t args[MAX_OPT_ARGS];
  630. int option;
  631. unsigned long long size = 0;
  632. enum { NO_SIZE, SIZE_STD, SIZE_PERCENT } setsize = NO_SIZE;
  633. if (!options)
  634. return 0;
  635. while ((p = strsep(&options, ",")) != NULL) {
  636. int token;
  637. if (!*p)
  638. continue;
  639. token = match_token(p, tokens, args);
  640. switch (token) {
  641. case Opt_uid:
  642. if (match_int(&args[0], &option))
  643. goto bad_val;
  644. pconfig->uid = option;
  645. break;
  646. case Opt_gid:
  647. if (match_int(&args[0], &option))
  648. goto bad_val;
  649. pconfig->gid = option;
  650. break;
  651. case Opt_mode:
  652. if (match_octal(&args[0], &option))
  653. goto bad_val;
  654. pconfig->mode = option & 01777U;
  655. break;
  656. case Opt_size: {
  657. /* memparse() will accept a K/M/G without a digit */
  658. if (!isdigit(*args[0].from))
  659. goto bad_val;
  660. size = memparse(args[0].from, &rest);
  661. setsize = SIZE_STD;
  662. if (*rest == '%')
  663. setsize = SIZE_PERCENT;
  664. break;
  665. }
  666. case Opt_nr_inodes:
  667. /* memparse() will accept a K/M/G without a digit */
  668. if (!isdigit(*args[0].from))
  669. goto bad_val;
  670. pconfig->nr_inodes = memparse(args[0].from, &rest);
  671. break;
  672. case Opt_pagesize: {
  673. unsigned long ps;
  674. ps = memparse(args[0].from, &rest);
  675. pconfig->hstate = size_to_hstate(ps);
  676. if (!pconfig->hstate) {
  677. printk(KERN_ERR
  678. "hugetlbfs: Unsupported page size %lu MB\n",
  679. ps >> 20);
  680. return -EINVAL;
  681. }
  682. break;
  683. }
  684. default:
  685. printk(KERN_ERR "hugetlbfs: Bad mount option: \"%s\"\n",
  686. p);
  687. return -EINVAL;
  688. break;
  689. }
  690. }
  691. /* Do size after hstate is set up */
  692. if (setsize > NO_SIZE) {
  693. struct hstate *h = pconfig->hstate;
  694. if (setsize == SIZE_PERCENT) {
  695. size <<= huge_page_shift(h);
  696. size *= h->max_huge_pages;
  697. do_div(size, 100);
  698. }
  699. pconfig->nr_blocks = (size >> huge_page_shift(h));
  700. }
  701. return 0;
  702. bad_val:
  703. printk(KERN_ERR "hugetlbfs: Bad value '%s' for mount option '%s'\n",
  704. args[0].from, p);
  705. return -EINVAL;
  706. }
  707. static int
  708. hugetlbfs_fill_super(struct super_block *sb, void *data, int silent)
  709. {
  710. struct inode * inode;
  711. struct dentry * root;
  712. int ret;
  713. struct hugetlbfs_config config;
  714. struct hugetlbfs_sb_info *sbinfo;
  715. save_mount_options(sb, data);
  716. config.nr_blocks = -1; /* No limit on size by default */
  717. config.nr_inodes = -1; /* No limit on number of inodes by default */
  718. config.uid = current_fsuid();
  719. config.gid = current_fsgid();
  720. config.mode = 0755;
  721. config.hstate = &default_hstate;
  722. ret = hugetlbfs_parse_options(data, &config);
  723. if (ret)
  724. return ret;
  725. sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
  726. if (!sbinfo)
  727. return -ENOMEM;
  728. sb->s_fs_info = sbinfo;
  729. sbinfo->hstate = config.hstate;
  730. spin_lock_init(&sbinfo->stat_lock);
  731. sbinfo->max_blocks = config.nr_blocks;
  732. sbinfo->free_blocks = config.nr_blocks;
  733. sbinfo->max_inodes = config.nr_inodes;
  734. sbinfo->free_inodes = config.nr_inodes;
  735. sb->s_maxbytes = MAX_LFS_FILESIZE;
  736. sb->s_blocksize = huge_page_size(config.hstate);
  737. sb->s_blocksize_bits = huge_page_shift(config.hstate);
  738. sb->s_magic = HUGETLBFS_MAGIC;
  739. sb->s_op = &hugetlbfs_ops;
  740. sb->s_time_gran = 1;
  741. inode = hugetlbfs_get_inode(sb, config.uid, config.gid,
  742. S_IFDIR | config.mode, 0);
  743. if (!inode)
  744. goto out_free;
  745. root = d_alloc_root(inode);
  746. if (!root) {
  747. iput(inode);
  748. goto out_free;
  749. }
  750. sb->s_root = root;
  751. return 0;
  752. out_free:
  753. kfree(sbinfo);
  754. return -ENOMEM;
  755. }
  756. int hugetlb_get_quota(struct address_space *mapping, long delta)
  757. {
  758. int ret = 0;
  759. struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(mapping->host->i_sb);
  760. if (sbinfo->free_blocks > -1) {
  761. spin_lock(&sbinfo->stat_lock);
  762. if (sbinfo->free_blocks - delta >= 0)
  763. sbinfo->free_blocks -= delta;
  764. else
  765. ret = -ENOMEM;
  766. spin_unlock(&sbinfo->stat_lock);
  767. }
  768. return ret;
  769. }
  770. void hugetlb_put_quota(struct address_space *mapping, long delta)
  771. {
  772. struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(mapping->host->i_sb);
  773. if (sbinfo->free_blocks > -1) {
  774. spin_lock(&sbinfo->stat_lock);
  775. sbinfo->free_blocks += delta;
  776. spin_unlock(&sbinfo->stat_lock);
  777. }
  778. }
  779. static int hugetlbfs_get_sb(struct file_system_type *fs_type,
  780. int flags, const char *dev_name, void *data, struct vfsmount *mnt)
  781. {
  782. return get_sb_nodev(fs_type, flags, data, hugetlbfs_fill_super, mnt);
  783. }
  784. static struct file_system_type hugetlbfs_fs_type = {
  785. .name = "hugetlbfs",
  786. .get_sb = hugetlbfs_get_sb,
  787. .kill_sb = kill_litter_super,
  788. };
  789. static struct vfsmount *hugetlbfs_vfsmount;
  790. static int can_do_hugetlb_shm(void)
  791. {
  792. return capable(CAP_IPC_LOCK) || in_group_p(sysctl_hugetlb_shm_group);
  793. }
  794. struct file *hugetlb_file_setup(const char *name, size_t size, int acctflag,
  795. struct user_struct **user, int creat_flags)
  796. {
  797. int error = -ENOMEM;
  798. struct file *file;
  799. struct inode *inode;
  800. struct path path;
  801. struct dentry *root;
  802. struct qstr quick_string;
  803. *user = NULL;
  804. if (!hugetlbfs_vfsmount)
  805. return ERR_PTR(-ENOENT);
  806. if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) {
  807. *user = current_user();
  808. if (user_shm_lock(size, *user)) {
  809. WARN_ONCE(1,
  810. "Using mlock ulimits for SHM_HUGETLB deprecated\n");
  811. } else {
  812. *user = NULL;
  813. return ERR_PTR(-EPERM);
  814. }
  815. }
  816. root = hugetlbfs_vfsmount->mnt_root;
  817. quick_string.name = name;
  818. quick_string.len = strlen(quick_string.name);
  819. quick_string.hash = 0;
  820. path.dentry = d_alloc(root, &quick_string);
  821. if (!path.dentry)
  822. goto out_shm_unlock;
  823. path.mnt = mntget(hugetlbfs_vfsmount);
  824. error = -ENOSPC;
  825. inode = hugetlbfs_get_inode(root->d_sb, current_fsuid(),
  826. current_fsgid(), S_IFREG | S_IRWXUGO, 0);
  827. if (!inode)
  828. goto out_dentry;
  829. error = -ENOMEM;
  830. if (hugetlb_reserve_pages(inode, 0,
  831. size >> huge_page_shift(hstate_inode(inode)), NULL,
  832. acctflag))
  833. goto out_inode;
  834. d_instantiate(path.dentry, inode);
  835. inode->i_size = size;
  836. inode->i_nlink = 0;
  837. error = -ENFILE;
  838. file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
  839. &hugetlbfs_file_operations);
  840. if (!file)
  841. goto out_dentry; /* inode is already attached */
  842. ima_counts_get(file);
  843. return file;
  844. out_inode:
  845. iput(inode);
  846. out_dentry:
  847. path_put(&path);
  848. out_shm_unlock:
  849. if (*user) {
  850. user_shm_unlock(size, *user);
  851. *user = NULL;
  852. }
  853. return ERR_PTR(error);
  854. }
  855. static int __init init_hugetlbfs_fs(void)
  856. {
  857. int error;
  858. struct vfsmount *vfsmount;
  859. error = bdi_init(&hugetlbfs_backing_dev_info);
  860. if (error)
  861. return error;
  862. hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
  863. sizeof(struct hugetlbfs_inode_info),
  864. 0, 0, init_once);
  865. if (hugetlbfs_inode_cachep == NULL)
  866. goto out2;
  867. error = register_filesystem(&hugetlbfs_fs_type);
  868. if (error)
  869. goto out;
  870. vfsmount = kern_mount(&hugetlbfs_fs_type);
  871. if (!IS_ERR(vfsmount)) {
  872. hugetlbfs_vfsmount = vfsmount;
  873. return 0;
  874. }
  875. error = PTR_ERR(vfsmount);
  876. out:
  877. if (error)
  878. kmem_cache_destroy(hugetlbfs_inode_cachep);
  879. out2:
  880. bdi_destroy(&hugetlbfs_backing_dev_info);
  881. return error;
  882. }
  883. static void __exit exit_hugetlbfs_fs(void)
  884. {
  885. kmem_cache_destroy(hugetlbfs_inode_cachep);
  886. unregister_filesystem(&hugetlbfs_fs_type);
  887. bdi_destroy(&hugetlbfs_backing_dev_info);
  888. }
  889. module_init(init_hugetlbfs_fs)
  890. module_exit(exit_hugetlbfs_fs)
  891. MODULE_LICENSE("GPL");