shmem.c 59 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359
  1. /*
  2. * Resizable virtual memory filesystem for Linux.
  3. *
  4. * Copyright (C) 2000 Linus Torvalds.
  5. * 2000 Transmeta Corp.
  6. * 2000-2001 Christoph Rohland
  7. * 2000-2001 SAP AG
  8. * 2002 Red Hat Inc.
  9. * Copyright (C) 2002-2005 Hugh Dickins.
  10. * Copyright (C) 2002-2005 VERITAS Software Corporation.
  11. * Copyright (C) 2004 Andi Kleen, SuSE Labs
  12. *
  13. * Extended attribute support for tmpfs:
  14. * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
  15. * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
  16. *
  17. * This file is released under the GPL.
  18. */
  19. /*
  20. * This virtual memory filesystem is heavily based on the ramfs. It
  21. * extends ramfs by the ability to use swap and honor resource limits
  22. * which makes it a completely usable filesystem.
  23. */
  24. #include <linux/config.h>
  25. #include <linux/module.h>
  26. #include <linux/init.h>
  27. #include <linux/devfs_fs_kernel.h>
  28. #include <linux/fs.h>
  29. #include <linux/mm.h>
  30. #include <linux/mman.h>
  31. #include <linux/file.h>
  32. #include <linux/swap.h>
  33. #include <linux/pagemap.h>
  34. #include <linux/string.h>
  35. #include <linux/slab.h>
  36. #include <linux/backing-dev.h>
  37. #include <linux/shmem_fs.h>
  38. #include <linux/mount.h>
  39. #include <linux/writeback.h>
  40. #include <linux/vfs.h>
  41. #include <linux/blkdev.h>
  42. #include <linux/security.h>
  43. #include <linux/swapops.h>
  44. #include <linux/mempolicy.h>
  45. #include <linux/namei.h>
  46. #include <linux/ctype.h>
  47. #include <linux/migrate.h>
  48. #include <asm/uaccess.h>
  49. #include <asm/div64.h>
  50. #include <asm/pgtable.h>
  51. /* This magic number is used in glibc for posix shared memory */
  52. #define TMPFS_MAGIC 0x01021994
  53. #define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
  54. #define ENTRIES_PER_PAGEPAGE (ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
  55. #define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
  56. #define SHMEM_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
  57. #define SHMEM_MAX_BYTES ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT)
  58. #define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
  59. /* info->flags needs VM_flags to handle pagein/truncate races efficiently */
  60. #define SHMEM_PAGEIN VM_READ
  61. #define SHMEM_TRUNCATE VM_WRITE
  62. /* Definition to limit shmem_truncate's steps between cond_rescheds */
  63. #define LATENCY_LIMIT 64
  64. /* Pretend that each entry is of this size in directory's i_size */
  65. #define BOGO_DIRENT_SIZE 20
  66. /* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */
  67. enum sgp_type {
  68. SGP_QUICK, /* don't try more than file page cache lookup */
  69. SGP_READ, /* don't exceed i_size, don't allocate page */
  70. SGP_CACHE, /* don't exceed i_size, may allocate page */
  71. SGP_WRITE, /* may exceed i_size, may allocate page */
  72. };
  73. static int shmem_getpage(struct inode *inode, unsigned long idx,
  74. struct page **pagep, enum sgp_type sgp, int *type);
  75. static inline struct page *shmem_dir_alloc(gfp_t gfp_mask)
  76. {
  77. /*
  78. * The above definition of ENTRIES_PER_PAGE, and the use of
  79. * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE:
  80. * might be reconsidered if it ever diverges from PAGE_SIZE.
  81. */
  82. return alloc_pages(gfp_mask, PAGE_CACHE_SHIFT-PAGE_SHIFT);
  83. }
  84. static inline void shmem_dir_free(struct page *page)
  85. {
  86. __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT);
  87. }
  88. static struct page **shmem_dir_map(struct page *page)
  89. {
  90. return (struct page **)kmap_atomic(page, KM_USER0);
  91. }
  92. static inline void shmem_dir_unmap(struct page **dir)
  93. {
  94. kunmap_atomic(dir, KM_USER0);
  95. }
  96. static swp_entry_t *shmem_swp_map(struct page *page)
  97. {
  98. return (swp_entry_t *)kmap_atomic(page, KM_USER1);
  99. }
  100. static inline void shmem_swp_balance_unmap(void)
  101. {
  102. /*
  103. * When passing a pointer to an i_direct entry, to code which
  104. * also handles indirect entries and so will shmem_swp_unmap,
  105. * we must arrange for the preempt count to remain in balance.
  106. * What kmap_atomic of a lowmem page does depends on config
  107. * and architecture, so pretend to kmap_atomic some lowmem page.
  108. */
  109. (void) kmap_atomic(ZERO_PAGE(0), KM_USER1);
  110. }
  111. static inline void shmem_swp_unmap(swp_entry_t *entry)
  112. {
  113. kunmap_atomic(entry, KM_USER1);
  114. }
  115. static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
  116. {
  117. return sb->s_fs_info;
  118. }
  119. /*
  120. * shmem_file_setup pre-accounts the whole fixed size of a VM object,
  121. * for shared memory and for shared anonymous (/dev/zero) mappings
  122. * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
  123. * consistent with the pre-accounting of private mappings ...
  124. */
  125. static inline int shmem_acct_size(unsigned long flags, loff_t size)
  126. {
  127. return (flags & VM_ACCOUNT)?
  128. security_vm_enough_memory(VM_ACCT(size)): 0;
  129. }
  130. static inline void shmem_unacct_size(unsigned long flags, loff_t size)
  131. {
  132. if (flags & VM_ACCOUNT)
  133. vm_unacct_memory(VM_ACCT(size));
  134. }
  135. /*
  136. * ... whereas tmpfs objects are accounted incrementally as
  137. * pages are allocated, in order to allow huge sparse files.
  138. * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
  139. * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
  140. */
  141. static inline int shmem_acct_block(unsigned long flags)
  142. {
  143. return (flags & VM_ACCOUNT)?
  144. 0: security_vm_enough_memory(VM_ACCT(PAGE_CACHE_SIZE));
  145. }
  146. static inline void shmem_unacct_blocks(unsigned long flags, long pages)
  147. {
  148. if (!(flags & VM_ACCOUNT))
  149. vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
  150. }
  151. static struct super_operations shmem_ops;
  152. static struct address_space_operations shmem_aops;
  153. static struct file_operations shmem_file_operations;
  154. static struct inode_operations shmem_inode_operations;
  155. static struct inode_operations shmem_dir_inode_operations;
  156. static struct vm_operations_struct shmem_vm_ops;
  157. static struct backing_dev_info shmem_backing_dev_info __read_mostly = {
  158. .ra_pages = 0, /* No readahead */
  159. .capabilities = BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_WRITEBACK,
  160. .unplug_io_fn = default_unplug_io_fn,
  161. };
  162. static LIST_HEAD(shmem_swaplist);
  163. static DEFINE_SPINLOCK(shmem_swaplist_lock);
  164. static void shmem_free_blocks(struct inode *inode, long pages)
  165. {
  166. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  167. if (sbinfo->max_blocks) {
  168. spin_lock(&sbinfo->stat_lock);
  169. sbinfo->free_blocks += pages;
  170. inode->i_blocks -= pages*BLOCKS_PER_PAGE;
  171. spin_unlock(&sbinfo->stat_lock);
  172. }
  173. }
  174. /*
  175. * shmem_recalc_inode - recalculate the size of an inode
  176. *
  177. * @inode: inode to recalc
  178. *
  179. * We have to calculate the free blocks since the mm can drop
  180. * undirtied hole pages behind our back.
  181. *
  182. * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
  183. * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
  184. *
  185. * It has to be called with the spinlock held.
  186. */
  187. static void shmem_recalc_inode(struct inode *inode)
  188. {
  189. struct shmem_inode_info *info = SHMEM_I(inode);
  190. long freed;
  191. freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
  192. if (freed > 0) {
  193. info->alloced -= freed;
  194. shmem_unacct_blocks(info->flags, freed);
  195. shmem_free_blocks(inode, freed);
  196. }
  197. }
  198. /*
  199. * shmem_swp_entry - find the swap vector position in the info structure
  200. *
  201. * @info: info structure for the inode
  202. * @index: index of the page to find
  203. * @page: optional page to add to the structure. Has to be preset to
  204. * all zeros
  205. *
  206. * If there is no space allocated yet it will return NULL when
  207. * page is NULL, else it will use the page for the needed block,
  208. * setting it to NULL on return to indicate that it has been used.
  209. *
  210. * The swap vector is organized the following way:
  211. *
  212. * There are SHMEM_NR_DIRECT entries directly stored in the
  213. * shmem_inode_info structure. So small files do not need an addional
  214. * allocation.
  215. *
  216. * For pages with index > SHMEM_NR_DIRECT there is the pointer
  217. * i_indirect which points to a page which holds in the first half
  218. * doubly indirect blocks, in the second half triple indirect blocks:
  219. *
  220. * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the
  221. * following layout (for SHMEM_NR_DIRECT == 16):
  222. *
  223. * i_indirect -> dir --> 16-19
  224. * | +-> 20-23
  225. * |
  226. * +-->dir2 --> 24-27
  227. * | +-> 28-31
  228. * | +-> 32-35
  229. * | +-> 36-39
  230. * |
  231. * +-->dir3 --> 40-43
  232. * +-> 44-47
  233. * +-> 48-51
  234. * +-> 52-55
  235. */
  236. static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page)
  237. {
  238. unsigned long offset;
  239. struct page **dir;
  240. struct page *subdir;
  241. if (index < SHMEM_NR_DIRECT) {
  242. shmem_swp_balance_unmap();
  243. return info->i_direct+index;
  244. }
  245. if (!info->i_indirect) {
  246. if (page) {
  247. info->i_indirect = *page;
  248. *page = NULL;
  249. }
  250. return NULL; /* need another page */
  251. }
  252. index -= SHMEM_NR_DIRECT;
  253. offset = index % ENTRIES_PER_PAGE;
  254. index /= ENTRIES_PER_PAGE;
  255. dir = shmem_dir_map(info->i_indirect);
  256. if (index >= ENTRIES_PER_PAGE/2) {
  257. index -= ENTRIES_PER_PAGE/2;
  258. dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE;
  259. index %= ENTRIES_PER_PAGE;
  260. subdir = *dir;
  261. if (!subdir) {
  262. if (page) {
  263. *dir = *page;
  264. *page = NULL;
  265. }
  266. shmem_dir_unmap(dir);
  267. return NULL; /* need another page */
  268. }
  269. shmem_dir_unmap(dir);
  270. dir = shmem_dir_map(subdir);
  271. }
  272. dir += index;
  273. subdir = *dir;
  274. if (!subdir) {
  275. if (!page || !(subdir = *page)) {
  276. shmem_dir_unmap(dir);
  277. return NULL; /* need a page */
  278. }
  279. *dir = subdir;
  280. *page = NULL;
  281. }
  282. shmem_dir_unmap(dir);
  283. return shmem_swp_map(subdir) + offset;
  284. }
  285. static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value)
  286. {
  287. long incdec = value? 1: -1;
  288. entry->val = value;
  289. info->swapped += incdec;
  290. if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) {
  291. struct page *page = kmap_atomic_to_page(entry);
  292. set_page_private(page, page_private(page) + incdec);
  293. }
  294. }
  295. /*
  296. * shmem_swp_alloc - get the position of the swap entry for the page.
  297. * If it does not exist allocate the entry.
  298. *
  299. * @info: info structure for the inode
  300. * @index: index of the page to find
  301. * @sgp: check and recheck i_size? skip allocation?
  302. */
  303. static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp)
  304. {
  305. struct inode *inode = &info->vfs_inode;
  306. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  307. struct page *page = NULL;
  308. swp_entry_t *entry;
  309. if (sgp != SGP_WRITE &&
  310. ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode))
  311. return ERR_PTR(-EINVAL);
  312. while (!(entry = shmem_swp_entry(info, index, &page))) {
  313. if (sgp == SGP_READ)
  314. return shmem_swp_map(ZERO_PAGE(0));
  315. /*
  316. * Test free_blocks against 1 not 0, since we have 1 data
  317. * page (and perhaps indirect index pages) yet to allocate:
  318. * a waste to allocate index if we cannot allocate data.
  319. */
  320. if (sbinfo->max_blocks) {
  321. spin_lock(&sbinfo->stat_lock);
  322. if (sbinfo->free_blocks <= 1) {
  323. spin_unlock(&sbinfo->stat_lock);
  324. return ERR_PTR(-ENOSPC);
  325. }
  326. sbinfo->free_blocks--;
  327. inode->i_blocks += BLOCKS_PER_PAGE;
  328. spin_unlock(&sbinfo->stat_lock);
  329. }
  330. spin_unlock(&info->lock);
  331. page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping) | __GFP_ZERO);
  332. if (page)
  333. set_page_private(page, 0);
  334. spin_lock(&info->lock);
  335. if (!page) {
  336. shmem_free_blocks(inode, 1);
  337. return ERR_PTR(-ENOMEM);
  338. }
  339. if (sgp != SGP_WRITE &&
  340. ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
  341. entry = ERR_PTR(-EINVAL);
  342. break;
  343. }
  344. if (info->next_index <= index)
  345. info->next_index = index + 1;
  346. }
  347. if (page) {
  348. /* another task gave its page, or truncated the file */
  349. shmem_free_blocks(inode, 1);
  350. shmem_dir_free(page);
  351. }
  352. if (info->next_index <= index && !IS_ERR(entry))
  353. info->next_index = index + 1;
  354. return entry;
  355. }
  356. /*
  357. * shmem_free_swp - free some swap entries in a directory
  358. *
  359. * @dir: pointer to the directory
  360. * @edir: pointer after last entry of the directory
  361. */
  362. static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir)
  363. {
  364. swp_entry_t *ptr;
  365. int freed = 0;
  366. for (ptr = dir; ptr < edir; ptr++) {
  367. if (ptr->val) {
  368. free_swap_and_cache(*ptr);
  369. *ptr = (swp_entry_t){0};
  370. freed++;
  371. }
  372. }
  373. return freed;
  374. }
  375. static int shmem_map_and_free_swp(struct page *subdir,
  376. int offset, int limit, struct page ***dir)
  377. {
  378. swp_entry_t *ptr;
  379. int freed = 0;
  380. ptr = shmem_swp_map(subdir);
  381. for (; offset < limit; offset += LATENCY_LIMIT) {
  382. int size = limit - offset;
  383. if (size > LATENCY_LIMIT)
  384. size = LATENCY_LIMIT;
  385. freed += shmem_free_swp(ptr+offset, ptr+offset+size);
  386. if (need_resched()) {
  387. shmem_swp_unmap(ptr);
  388. if (*dir) {
  389. shmem_dir_unmap(*dir);
  390. *dir = NULL;
  391. }
  392. cond_resched();
  393. ptr = shmem_swp_map(subdir);
  394. }
  395. }
  396. shmem_swp_unmap(ptr);
  397. return freed;
  398. }
  399. static void shmem_free_pages(struct list_head *next)
  400. {
  401. struct page *page;
  402. int freed = 0;
  403. do {
  404. page = container_of(next, struct page, lru);
  405. next = next->next;
  406. shmem_dir_free(page);
  407. freed++;
  408. if (freed >= LATENCY_LIMIT) {
  409. cond_resched();
  410. freed = 0;
  411. }
  412. } while (next);
  413. }
  414. static void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end)
  415. {
  416. struct shmem_inode_info *info = SHMEM_I(inode);
  417. unsigned long idx;
  418. unsigned long size;
  419. unsigned long limit;
  420. unsigned long stage;
  421. unsigned long diroff;
  422. struct page **dir;
  423. struct page *topdir;
  424. struct page *middir;
  425. struct page *subdir;
  426. swp_entry_t *ptr;
  427. LIST_HEAD(pages_to_free);
  428. long nr_pages_to_free = 0;
  429. long nr_swaps_freed = 0;
  430. int offset;
  431. int freed;
  432. int punch_hole = 0;
  433. inode->i_ctime = inode->i_mtime = CURRENT_TIME;
  434. idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  435. if (idx >= info->next_index)
  436. return;
  437. spin_lock(&info->lock);
  438. info->flags |= SHMEM_TRUNCATE;
  439. if (likely(end == (loff_t) -1)) {
  440. limit = info->next_index;
  441. info->next_index = idx;
  442. } else {
  443. limit = (end + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  444. if (limit > info->next_index)
  445. limit = info->next_index;
  446. punch_hole = 1;
  447. }
  448. topdir = info->i_indirect;
  449. if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) {
  450. info->i_indirect = NULL;
  451. nr_pages_to_free++;
  452. list_add(&topdir->lru, &pages_to_free);
  453. }
  454. spin_unlock(&info->lock);
  455. if (info->swapped && idx < SHMEM_NR_DIRECT) {
  456. ptr = info->i_direct;
  457. size = limit;
  458. if (size > SHMEM_NR_DIRECT)
  459. size = SHMEM_NR_DIRECT;
  460. nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size);
  461. }
  462. if (!topdir)
  463. goto done2;
  464. BUG_ON(limit <= SHMEM_NR_DIRECT);
  465. limit -= SHMEM_NR_DIRECT;
  466. idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0;
  467. offset = idx % ENTRIES_PER_PAGE;
  468. idx -= offset;
  469. dir = shmem_dir_map(topdir);
  470. stage = ENTRIES_PER_PAGEPAGE/2;
  471. if (idx < ENTRIES_PER_PAGEPAGE/2) {
  472. middir = topdir;
  473. diroff = idx/ENTRIES_PER_PAGE;
  474. } else {
  475. dir += ENTRIES_PER_PAGE/2;
  476. dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE;
  477. while (stage <= idx)
  478. stage += ENTRIES_PER_PAGEPAGE;
  479. middir = *dir;
  480. if (*dir) {
  481. diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) %
  482. ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE;
  483. if (!diroff && !offset) {
  484. *dir = NULL;
  485. nr_pages_to_free++;
  486. list_add(&middir->lru, &pages_to_free);
  487. }
  488. shmem_dir_unmap(dir);
  489. dir = shmem_dir_map(middir);
  490. } else {
  491. diroff = 0;
  492. offset = 0;
  493. idx = stage;
  494. }
  495. }
  496. for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) {
  497. if (unlikely(idx == stage)) {
  498. shmem_dir_unmap(dir);
  499. dir = shmem_dir_map(topdir) +
  500. ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
  501. while (!*dir) {
  502. dir++;
  503. idx += ENTRIES_PER_PAGEPAGE;
  504. if (idx >= limit)
  505. goto done1;
  506. }
  507. stage = idx + ENTRIES_PER_PAGEPAGE;
  508. middir = *dir;
  509. *dir = NULL;
  510. nr_pages_to_free++;
  511. list_add(&middir->lru, &pages_to_free);
  512. shmem_dir_unmap(dir);
  513. cond_resched();
  514. dir = shmem_dir_map(middir);
  515. diroff = 0;
  516. }
  517. subdir = dir[diroff];
  518. if (subdir && page_private(subdir)) {
  519. size = limit - idx;
  520. if (size > ENTRIES_PER_PAGE)
  521. size = ENTRIES_PER_PAGE;
  522. freed = shmem_map_and_free_swp(subdir,
  523. offset, size, &dir);
  524. if (!dir)
  525. dir = shmem_dir_map(middir);
  526. nr_swaps_freed += freed;
  527. if (offset)
  528. spin_lock(&info->lock);
  529. set_page_private(subdir, page_private(subdir) - freed);
  530. if (offset)
  531. spin_unlock(&info->lock);
  532. if (!punch_hole)
  533. BUG_ON(page_private(subdir) > offset);
  534. }
  535. if (offset)
  536. offset = 0;
  537. else if (subdir && !page_private(subdir)) {
  538. dir[diroff] = NULL;
  539. nr_pages_to_free++;
  540. list_add(&subdir->lru, &pages_to_free);
  541. }
  542. }
  543. done1:
  544. shmem_dir_unmap(dir);
  545. done2:
  546. if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) {
  547. /*
  548. * Call truncate_inode_pages again: racing shmem_unuse_inode
  549. * may have swizzled a page in from swap since vmtruncate or
  550. * generic_delete_inode did it, before we lowered next_index.
  551. * Also, though shmem_getpage checks i_size before adding to
  552. * cache, no recheck after: so fix the narrow window there too.
  553. */
  554. truncate_inode_pages_range(inode->i_mapping, start, end);
  555. }
  556. spin_lock(&info->lock);
  557. info->flags &= ~SHMEM_TRUNCATE;
  558. info->swapped -= nr_swaps_freed;
  559. if (nr_pages_to_free)
  560. shmem_free_blocks(inode, nr_pages_to_free);
  561. shmem_recalc_inode(inode);
  562. spin_unlock(&info->lock);
  563. /*
  564. * Empty swap vector directory pages to be freed?
  565. */
  566. if (!list_empty(&pages_to_free)) {
  567. pages_to_free.prev->next = NULL;
  568. shmem_free_pages(pages_to_free.next);
  569. }
  570. }
  571. static void shmem_truncate(struct inode *inode)
  572. {
  573. shmem_truncate_range(inode, inode->i_size, (loff_t)-1);
  574. }
  575. static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
  576. {
  577. struct inode *inode = dentry->d_inode;
  578. struct page *page = NULL;
  579. int error;
  580. if (attr->ia_valid & ATTR_SIZE) {
  581. if (attr->ia_size < inode->i_size) {
  582. /*
  583. * If truncating down to a partial page, then
  584. * if that page is already allocated, hold it
  585. * in memory until the truncation is over, so
  586. * truncate_partial_page cannnot miss it were
  587. * it assigned to swap.
  588. */
  589. if (attr->ia_size & (PAGE_CACHE_SIZE-1)) {
  590. (void) shmem_getpage(inode,
  591. attr->ia_size>>PAGE_CACHE_SHIFT,
  592. &page, SGP_READ, NULL);
  593. }
  594. /*
  595. * Reset SHMEM_PAGEIN flag so that shmem_truncate can
  596. * detect if any pages might have been added to cache
  597. * after truncate_inode_pages. But we needn't bother
  598. * if it's being fully truncated to zero-length: the
  599. * nrpages check is efficient enough in that case.
  600. */
  601. if (attr->ia_size) {
  602. struct shmem_inode_info *info = SHMEM_I(inode);
  603. spin_lock(&info->lock);
  604. info->flags &= ~SHMEM_PAGEIN;
  605. spin_unlock(&info->lock);
  606. }
  607. }
  608. }
  609. error = inode_change_ok(inode, attr);
  610. if (!error)
  611. error = inode_setattr(inode, attr);
  612. if (page)
  613. page_cache_release(page);
  614. return error;
  615. }
  616. static void shmem_delete_inode(struct inode *inode)
  617. {
  618. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  619. struct shmem_inode_info *info = SHMEM_I(inode);
  620. if (inode->i_op->truncate == shmem_truncate) {
  621. truncate_inode_pages(inode->i_mapping, 0);
  622. shmem_unacct_size(info->flags, inode->i_size);
  623. inode->i_size = 0;
  624. shmem_truncate(inode);
  625. if (!list_empty(&info->swaplist)) {
  626. spin_lock(&shmem_swaplist_lock);
  627. list_del_init(&info->swaplist);
  628. spin_unlock(&shmem_swaplist_lock);
  629. }
  630. }
  631. BUG_ON(inode->i_blocks);
  632. if (sbinfo->max_inodes) {
  633. spin_lock(&sbinfo->stat_lock);
  634. sbinfo->free_inodes++;
  635. spin_unlock(&sbinfo->stat_lock);
  636. }
  637. clear_inode(inode);
  638. }
  639. static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
  640. {
  641. swp_entry_t *ptr;
  642. for (ptr = dir; ptr < edir; ptr++) {
  643. if (ptr->val == entry.val)
  644. return ptr - dir;
  645. }
  646. return -1;
  647. }
  648. static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
  649. {
  650. struct inode *inode;
  651. unsigned long idx;
  652. unsigned long size;
  653. unsigned long limit;
  654. unsigned long stage;
  655. struct page **dir;
  656. struct page *subdir;
  657. swp_entry_t *ptr;
  658. int offset;
  659. idx = 0;
  660. ptr = info->i_direct;
  661. spin_lock(&info->lock);
  662. limit = info->next_index;
  663. size = limit;
  664. if (size > SHMEM_NR_DIRECT)
  665. size = SHMEM_NR_DIRECT;
  666. offset = shmem_find_swp(entry, ptr, ptr+size);
  667. if (offset >= 0) {
  668. shmem_swp_balance_unmap();
  669. goto found;
  670. }
  671. if (!info->i_indirect)
  672. goto lost2;
  673. dir = shmem_dir_map(info->i_indirect);
  674. stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2;
  675. for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
  676. if (unlikely(idx == stage)) {
  677. shmem_dir_unmap(dir-1);
  678. dir = shmem_dir_map(info->i_indirect) +
  679. ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
  680. while (!*dir) {
  681. dir++;
  682. idx += ENTRIES_PER_PAGEPAGE;
  683. if (idx >= limit)
  684. goto lost1;
  685. }
  686. stage = idx + ENTRIES_PER_PAGEPAGE;
  687. subdir = *dir;
  688. shmem_dir_unmap(dir);
  689. dir = shmem_dir_map(subdir);
  690. }
  691. subdir = *dir;
  692. if (subdir && page_private(subdir)) {
  693. ptr = shmem_swp_map(subdir);
  694. size = limit - idx;
  695. if (size > ENTRIES_PER_PAGE)
  696. size = ENTRIES_PER_PAGE;
  697. offset = shmem_find_swp(entry, ptr, ptr+size);
  698. if (offset >= 0) {
  699. shmem_dir_unmap(dir);
  700. goto found;
  701. }
  702. shmem_swp_unmap(ptr);
  703. }
  704. }
  705. lost1:
  706. shmem_dir_unmap(dir-1);
  707. lost2:
  708. spin_unlock(&info->lock);
  709. return 0;
  710. found:
  711. idx += offset;
  712. inode = &info->vfs_inode;
  713. if (move_from_swap_cache(page, idx, inode->i_mapping) == 0) {
  714. info->flags |= SHMEM_PAGEIN;
  715. shmem_swp_set(info, ptr + offset, 0);
  716. }
  717. shmem_swp_unmap(ptr);
  718. spin_unlock(&info->lock);
  719. /*
  720. * Decrement swap count even when the entry is left behind:
  721. * try_to_unuse will skip over mms, then reincrement count.
  722. */
  723. swap_free(entry);
  724. return 1;
  725. }
  726. /*
  727. * shmem_unuse() search for an eventually swapped out shmem page.
  728. */
  729. int shmem_unuse(swp_entry_t entry, struct page *page)
  730. {
  731. struct list_head *p, *next;
  732. struct shmem_inode_info *info;
  733. int found = 0;
  734. spin_lock(&shmem_swaplist_lock);
  735. list_for_each_safe(p, next, &shmem_swaplist) {
  736. info = list_entry(p, struct shmem_inode_info, swaplist);
  737. if (!info->swapped)
  738. list_del_init(&info->swaplist);
  739. else if (shmem_unuse_inode(info, entry, page)) {
  740. /* move head to start search for next from here */
  741. list_move_tail(&shmem_swaplist, &info->swaplist);
  742. found = 1;
  743. break;
  744. }
  745. }
  746. spin_unlock(&shmem_swaplist_lock);
  747. return found;
  748. }
  749. /*
  750. * Move the page from the page cache to the swap cache.
  751. */
  752. static int shmem_writepage(struct page *page, struct writeback_control *wbc)
  753. {
  754. struct shmem_inode_info *info;
  755. swp_entry_t *entry, swap;
  756. struct address_space *mapping;
  757. unsigned long index;
  758. struct inode *inode;
  759. BUG_ON(!PageLocked(page));
  760. BUG_ON(page_mapped(page));
  761. mapping = page->mapping;
  762. index = page->index;
  763. inode = mapping->host;
  764. info = SHMEM_I(inode);
  765. if (info->flags & VM_LOCKED)
  766. goto redirty;
  767. swap = get_swap_page();
  768. if (!swap.val)
  769. goto redirty;
  770. spin_lock(&info->lock);
  771. shmem_recalc_inode(inode);
  772. if (index >= info->next_index) {
  773. BUG_ON(!(info->flags & SHMEM_TRUNCATE));
  774. goto unlock;
  775. }
  776. entry = shmem_swp_entry(info, index, NULL);
  777. BUG_ON(!entry);
  778. BUG_ON(entry->val);
  779. if (move_to_swap_cache(page, swap) == 0) {
  780. shmem_swp_set(info, entry, swap.val);
  781. shmem_swp_unmap(entry);
  782. spin_unlock(&info->lock);
  783. if (list_empty(&info->swaplist)) {
  784. spin_lock(&shmem_swaplist_lock);
  785. /* move instead of add in case we're racing */
  786. list_move_tail(&info->swaplist, &shmem_swaplist);
  787. spin_unlock(&shmem_swaplist_lock);
  788. }
  789. unlock_page(page);
  790. return 0;
  791. }
  792. shmem_swp_unmap(entry);
  793. unlock:
  794. spin_unlock(&info->lock);
  795. swap_free(swap);
  796. redirty:
  797. set_page_dirty(page);
  798. return AOP_WRITEPAGE_ACTIVATE; /* Return with the page locked */
  799. }
  800. #ifdef CONFIG_NUMA
  801. static inline int shmem_parse_mpol(char *value, int *policy, nodemask_t *policy_nodes)
  802. {
  803. char *nodelist = strchr(value, ':');
  804. int err = 1;
  805. if (nodelist) {
  806. /* NUL-terminate policy string */
  807. *nodelist++ = '\0';
  808. if (nodelist_parse(nodelist, *policy_nodes))
  809. goto out;
  810. }
  811. if (!strcmp(value, "default")) {
  812. *policy = MPOL_DEFAULT;
  813. /* Don't allow a nodelist */
  814. if (!nodelist)
  815. err = 0;
  816. } else if (!strcmp(value, "prefer")) {
  817. *policy = MPOL_PREFERRED;
  818. /* Insist on a nodelist of one node only */
  819. if (nodelist) {
  820. char *rest = nodelist;
  821. while (isdigit(*rest))
  822. rest++;
  823. if (!*rest)
  824. err = 0;
  825. }
  826. } else if (!strcmp(value, "bind")) {
  827. *policy = MPOL_BIND;
  828. /* Insist on a nodelist */
  829. if (nodelist)
  830. err = 0;
  831. } else if (!strcmp(value, "interleave")) {
  832. *policy = MPOL_INTERLEAVE;
  833. /* Default to nodes online if no nodelist */
  834. if (!nodelist)
  835. *policy_nodes = node_online_map;
  836. err = 0;
  837. }
  838. out:
  839. /* Restore string for error message */
  840. if (nodelist)
  841. *--nodelist = ':';
  842. return err;
  843. }
  844. static struct page *shmem_swapin_async(struct shared_policy *p,
  845. swp_entry_t entry, unsigned long idx)
  846. {
  847. struct page *page;
  848. struct vm_area_struct pvma;
  849. /* Create a pseudo vma that just contains the policy */
  850. memset(&pvma, 0, sizeof(struct vm_area_struct));
  851. pvma.vm_end = PAGE_SIZE;
  852. pvma.vm_pgoff = idx;
  853. pvma.vm_policy = mpol_shared_policy_lookup(p, idx);
  854. page = read_swap_cache_async(entry, &pvma, 0);
  855. mpol_free(pvma.vm_policy);
  856. return page;
  857. }
  858. struct page *shmem_swapin(struct shmem_inode_info *info, swp_entry_t entry,
  859. unsigned long idx)
  860. {
  861. struct shared_policy *p = &info->policy;
  862. int i, num;
  863. struct page *page;
  864. unsigned long offset;
  865. num = valid_swaphandles(entry, &offset);
  866. for (i = 0; i < num; offset++, i++) {
  867. page = shmem_swapin_async(p,
  868. swp_entry(swp_type(entry), offset), idx);
  869. if (!page)
  870. break;
  871. page_cache_release(page);
  872. }
  873. lru_add_drain(); /* Push any new pages onto the LRU now */
  874. return shmem_swapin_async(p, entry, idx);
  875. }
  876. static struct page *
  877. shmem_alloc_page(gfp_t gfp, struct shmem_inode_info *info,
  878. unsigned long idx)
  879. {
  880. struct vm_area_struct pvma;
  881. struct page *page;
  882. memset(&pvma, 0, sizeof(struct vm_area_struct));
  883. pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
  884. pvma.vm_pgoff = idx;
  885. pvma.vm_end = PAGE_SIZE;
  886. page = alloc_page_vma(gfp | __GFP_ZERO, &pvma, 0);
  887. mpol_free(pvma.vm_policy);
  888. return page;
  889. }
  890. #else
  891. static inline int shmem_parse_mpol(char *value, int *policy, nodemask_t *policy_nodes)
  892. {
  893. return 1;
  894. }
  895. static inline struct page *
  896. shmem_swapin(struct shmem_inode_info *info,swp_entry_t entry,unsigned long idx)
  897. {
  898. swapin_readahead(entry, 0, NULL);
  899. return read_swap_cache_async(entry, NULL, 0);
  900. }
  901. static inline struct page *
  902. shmem_alloc_page(gfp_t gfp,struct shmem_inode_info *info, unsigned long idx)
  903. {
  904. return alloc_page(gfp | __GFP_ZERO);
  905. }
  906. #endif
  907. /*
  908. * shmem_getpage - either get the page from swap or allocate a new one
  909. *
  910. * If we allocate a new one we do not mark it dirty. That's up to the
  911. * vm. If we swap it in we mark it dirty since we also free the swap
  912. * entry since a page cannot live in both the swap and page cache
  913. */
  914. static int shmem_getpage(struct inode *inode, unsigned long idx,
  915. struct page **pagep, enum sgp_type sgp, int *type)
  916. {
  917. struct address_space *mapping = inode->i_mapping;
  918. struct shmem_inode_info *info = SHMEM_I(inode);
  919. struct shmem_sb_info *sbinfo;
  920. struct page *filepage = *pagep;
  921. struct page *swappage;
  922. swp_entry_t *entry;
  923. swp_entry_t swap;
  924. int error;
  925. if (idx >= SHMEM_MAX_INDEX)
  926. return -EFBIG;
  927. /*
  928. * Normally, filepage is NULL on entry, and either found
  929. * uptodate immediately, or allocated and zeroed, or read
  930. * in under swappage, which is then assigned to filepage.
  931. * But shmem_prepare_write passes in a locked filepage,
  932. * which may be found not uptodate by other callers too,
  933. * and may need to be copied from the swappage read in.
  934. */
  935. repeat:
  936. if (!filepage)
  937. filepage = find_lock_page(mapping, idx);
  938. if (filepage && PageUptodate(filepage))
  939. goto done;
  940. error = 0;
  941. if (sgp == SGP_QUICK)
  942. goto failed;
  943. spin_lock(&info->lock);
  944. shmem_recalc_inode(inode);
  945. entry = shmem_swp_alloc(info, idx, sgp);
  946. if (IS_ERR(entry)) {
  947. spin_unlock(&info->lock);
  948. error = PTR_ERR(entry);
  949. goto failed;
  950. }
  951. swap = *entry;
  952. if (swap.val) {
  953. /* Look it up and read it in.. */
  954. swappage = lookup_swap_cache(swap);
  955. if (!swappage) {
  956. shmem_swp_unmap(entry);
  957. spin_unlock(&info->lock);
  958. /* here we actually do the io */
  959. if (type && *type == VM_FAULT_MINOR) {
  960. inc_page_state(pgmajfault);
  961. *type = VM_FAULT_MAJOR;
  962. }
  963. swappage = shmem_swapin(info, swap, idx);
  964. if (!swappage) {
  965. spin_lock(&info->lock);
  966. entry = shmem_swp_alloc(info, idx, sgp);
  967. if (IS_ERR(entry))
  968. error = PTR_ERR(entry);
  969. else {
  970. if (entry->val == swap.val)
  971. error = -ENOMEM;
  972. shmem_swp_unmap(entry);
  973. }
  974. spin_unlock(&info->lock);
  975. if (error)
  976. goto failed;
  977. goto repeat;
  978. }
  979. wait_on_page_locked(swappage);
  980. page_cache_release(swappage);
  981. goto repeat;
  982. }
  983. /* We have to do this with page locked to prevent races */
  984. if (TestSetPageLocked(swappage)) {
  985. shmem_swp_unmap(entry);
  986. spin_unlock(&info->lock);
  987. wait_on_page_locked(swappage);
  988. page_cache_release(swappage);
  989. goto repeat;
  990. }
  991. if (PageWriteback(swappage)) {
  992. shmem_swp_unmap(entry);
  993. spin_unlock(&info->lock);
  994. wait_on_page_writeback(swappage);
  995. unlock_page(swappage);
  996. page_cache_release(swappage);
  997. goto repeat;
  998. }
  999. if (!PageUptodate(swappage)) {
  1000. shmem_swp_unmap(entry);
  1001. spin_unlock(&info->lock);
  1002. unlock_page(swappage);
  1003. page_cache_release(swappage);
  1004. error = -EIO;
  1005. goto failed;
  1006. }
  1007. if (filepage) {
  1008. shmem_swp_set(info, entry, 0);
  1009. shmem_swp_unmap(entry);
  1010. delete_from_swap_cache(swappage);
  1011. spin_unlock(&info->lock);
  1012. copy_highpage(filepage, swappage);
  1013. unlock_page(swappage);
  1014. page_cache_release(swappage);
  1015. flush_dcache_page(filepage);
  1016. SetPageUptodate(filepage);
  1017. set_page_dirty(filepage);
  1018. swap_free(swap);
  1019. } else if (!(error = move_from_swap_cache(
  1020. swappage, idx, mapping))) {
  1021. info->flags |= SHMEM_PAGEIN;
  1022. shmem_swp_set(info, entry, 0);
  1023. shmem_swp_unmap(entry);
  1024. spin_unlock(&info->lock);
  1025. filepage = swappage;
  1026. swap_free(swap);
  1027. } else {
  1028. shmem_swp_unmap(entry);
  1029. spin_unlock(&info->lock);
  1030. unlock_page(swappage);
  1031. page_cache_release(swappage);
  1032. if (error == -ENOMEM) {
  1033. /* let kswapd refresh zone for GFP_ATOMICs */
  1034. blk_congestion_wait(WRITE, HZ/50);
  1035. }
  1036. goto repeat;
  1037. }
  1038. } else if (sgp == SGP_READ && !filepage) {
  1039. shmem_swp_unmap(entry);
  1040. filepage = find_get_page(mapping, idx);
  1041. if (filepage &&
  1042. (!PageUptodate(filepage) || TestSetPageLocked(filepage))) {
  1043. spin_unlock(&info->lock);
  1044. wait_on_page_locked(filepage);
  1045. page_cache_release(filepage);
  1046. filepage = NULL;
  1047. goto repeat;
  1048. }
  1049. spin_unlock(&info->lock);
  1050. } else {
  1051. shmem_swp_unmap(entry);
  1052. sbinfo = SHMEM_SB(inode->i_sb);
  1053. if (sbinfo->max_blocks) {
  1054. spin_lock(&sbinfo->stat_lock);
  1055. if (sbinfo->free_blocks == 0 ||
  1056. shmem_acct_block(info->flags)) {
  1057. spin_unlock(&sbinfo->stat_lock);
  1058. spin_unlock(&info->lock);
  1059. error = -ENOSPC;
  1060. goto failed;
  1061. }
  1062. sbinfo->free_blocks--;
  1063. inode->i_blocks += BLOCKS_PER_PAGE;
  1064. spin_unlock(&sbinfo->stat_lock);
  1065. } else if (shmem_acct_block(info->flags)) {
  1066. spin_unlock(&info->lock);
  1067. error = -ENOSPC;
  1068. goto failed;
  1069. }
  1070. if (!filepage) {
  1071. spin_unlock(&info->lock);
  1072. filepage = shmem_alloc_page(mapping_gfp_mask(mapping),
  1073. info,
  1074. idx);
  1075. if (!filepage) {
  1076. shmem_unacct_blocks(info->flags, 1);
  1077. shmem_free_blocks(inode, 1);
  1078. error = -ENOMEM;
  1079. goto failed;
  1080. }
  1081. spin_lock(&info->lock);
  1082. entry = shmem_swp_alloc(info, idx, sgp);
  1083. if (IS_ERR(entry))
  1084. error = PTR_ERR(entry);
  1085. else {
  1086. swap = *entry;
  1087. shmem_swp_unmap(entry);
  1088. }
  1089. if (error || swap.val || 0 != add_to_page_cache_lru(
  1090. filepage, mapping, idx, GFP_ATOMIC)) {
  1091. spin_unlock(&info->lock);
  1092. page_cache_release(filepage);
  1093. shmem_unacct_blocks(info->flags, 1);
  1094. shmem_free_blocks(inode, 1);
  1095. filepage = NULL;
  1096. if (error)
  1097. goto failed;
  1098. goto repeat;
  1099. }
  1100. info->flags |= SHMEM_PAGEIN;
  1101. }
  1102. info->alloced++;
  1103. spin_unlock(&info->lock);
  1104. flush_dcache_page(filepage);
  1105. SetPageUptodate(filepage);
  1106. }
  1107. done:
  1108. if (*pagep != filepage) {
  1109. unlock_page(filepage);
  1110. *pagep = filepage;
  1111. }
  1112. return 0;
  1113. failed:
  1114. if (*pagep != filepage) {
  1115. unlock_page(filepage);
  1116. page_cache_release(filepage);
  1117. }
  1118. return error;
  1119. }
  1120. struct page *shmem_nopage(struct vm_area_struct *vma, unsigned long address, int *type)
  1121. {
  1122. struct inode *inode = vma->vm_file->f_dentry->d_inode;
  1123. struct page *page = NULL;
  1124. unsigned long idx;
  1125. int error;
  1126. idx = (address - vma->vm_start) >> PAGE_SHIFT;
  1127. idx += vma->vm_pgoff;
  1128. idx >>= PAGE_CACHE_SHIFT - PAGE_SHIFT;
  1129. if (((loff_t) idx << PAGE_CACHE_SHIFT) >= i_size_read(inode))
  1130. return NOPAGE_SIGBUS;
  1131. error = shmem_getpage(inode, idx, &page, SGP_CACHE, type);
  1132. if (error)
  1133. return (error == -ENOMEM)? NOPAGE_OOM: NOPAGE_SIGBUS;
  1134. mark_page_accessed(page);
  1135. return page;
  1136. }
  1137. static int shmem_populate(struct vm_area_struct *vma,
  1138. unsigned long addr, unsigned long len,
  1139. pgprot_t prot, unsigned long pgoff, int nonblock)
  1140. {
  1141. struct inode *inode = vma->vm_file->f_dentry->d_inode;
  1142. struct mm_struct *mm = vma->vm_mm;
  1143. enum sgp_type sgp = nonblock? SGP_QUICK: SGP_CACHE;
  1144. unsigned long size;
  1145. size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1146. if (pgoff >= size || pgoff + (len >> PAGE_SHIFT) > size)
  1147. return -EINVAL;
  1148. while ((long) len > 0) {
  1149. struct page *page = NULL;
  1150. int err;
  1151. /*
  1152. * Will need changing if PAGE_CACHE_SIZE != PAGE_SIZE
  1153. */
  1154. err = shmem_getpage(inode, pgoff, &page, sgp, NULL);
  1155. if (err)
  1156. return err;
  1157. /* Page may still be null, but only if nonblock was set. */
  1158. if (page) {
  1159. mark_page_accessed(page);
  1160. err = install_page(mm, vma, addr, page, prot);
  1161. if (err) {
  1162. page_cache_release(page);
  1163. return err;
  1164. }
  1165. } else if (vma->vm_flags & VM_NONLINEAR) {
  1166. /* No page was found just because we can't read it in
  1167. * now (being here implies nonblock != 0), but the page
  1168. * may exist, so set the PTE to fault it in later. */
  1169. err = install_file_pte(mm, vma, addr, pgoff, prot);
  1170. if (err)
  1171. return err;
  1172. }
  1173. len -= PAGE_SIZE;
  1174. addr += PAGE_SIZE;
  1175. pgoff++;
  1176. }
  1177. return 0;
  1178. }
  1179. #ifdef CONFIG_NUMA
  1180. int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
  1181. {
  1182. struct inode *i = vma->vm_file->f_dentry->d_inode;
  1183. return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new);
  1184. }
  1185. struct mempolicy *
  1186. shmem_get_policy(struct vm_area_struct *vma, unsigned long addr)
  1187. {
  1188. struct inode *i = vma->vm_file->f_dentry->d_inode;
  1189. unsigned long idx;
  1190. idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
  1191. return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx);
  1192. }
  1193. #endif
  1194. int shmem_lock(struct file *file, int lock, struct user_struct *user)
  1195. {
  1196. struct inode *inode = file->f_dentry->d_inode;
  1197. struct shmem_inode_info *info = SHMEM_I(inode);
  1198. int retval = -ENOMEM;
  1199. spin_lock(&info->lock);
  1200. if (lock && !(info->flags & VM_LOCKED)) {
  1201. if (!user_shm_lock(inode->i_size, user))
  1202. goto out_nomem;
  1203. info->flags |= VM_LOCKED;
  1204. }
  1205. if (!lock && (info->flags & VM_LOCKED) && user) {
  1206. user_shm_unlock(inode->i_size, user);
  1207. info->flags &= ~VM_LOCKED;
  1208. }
  1209. retval = 0;
  1210. out_nomem:
  1211. spin_unlock(&info->lock);
  1212. return retval;
  1213. }
  1214. int shmem_mmap(struct file *file, struct vm_area_struct *vma)
  1215. {
  1216. file_accessed(file);
  1217. vma->vm_ops = &shmem_vm_ops;
  1218. return 0;
  1219. }
  1220. static struct inode *
  1221. shmem_get_inode(struct super_block *sb, int mode, dev_t dev)
  1222. {
  1223. struct inode *inode;
  1224. struct shmem_inode_info *info;
  1225. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  1226. if (sbinfo->max_inodes) {
  1227. spin_lock(&sbinfo->stat_lock);
  1228. if (!sbinfo->free_inodes) {
  1229. spin_unlock(&sbinfo->stat_lock);
  1230. return NULL;
  1231. }
  1232. sbinfo->free_inodes--;
  1233. spin_unlock(&sbinfo->stat_lock);
  1234. }
  1235. inode = new_inode(sb);
  1236. if (inode) {
  1237. inode->i_mode = mode;
  1238. inode->i_uid = current->fsuid;
  1239. inode->i_gid = current->fsgid;
  1240. inode->i_blksize = PAGE_CACHE_SIZE;
  1241. inode->i_blocks = 0;
  1242. inode->i_mapping->a_ops = &shmem_aops;
  1243. inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
  1244. inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
  1245. info = SHMEM_I(inode);
  1246. memset(info, 0, (char *)inode - (char *)info);
  1247. spin_lock_init(&info->lock);
  1248. INIT_LIST_HEAD(&info->swaplist);
  1249. switch (mode & S_IFMT) {
  1250. default:
  1251. init_special_inode(inode, mode, dev);
  1252. break;
  1253. case S_IFREG:
  1254. inode->i_op = &shmem_inode_operations;
  1255. inode->i_fop = &shmem_file_operations;
  1256. mpol_shared_policy_init(&info->policy, sbinfo->policy,
  1257. &sbinfo->policy_nodes);
  1258. break;
  1259. case S_IFDIR:
  1260. inode->i_nlink++;
  1261. /* Some things misbehave if size == 0 on a directory */
  1262. inode->i_size = 2 * BOGO_DIRENT_SIZE;
  1263. inode->i_op = &shmem_dir_inode_operations;
  1264. inode->i_fop = &simple_dir_operations;
  1265. break;
  1266. case S_IFLNK:
  1267. /*
  1268. * Must not load anything in the rbtree,
  1269. * mpol_free_shared_policy will not be called.
  1270. */
  1271. mpol_shared_policy_init(&info->policy, MPOL_DEFAULT,
  1272. NULL);
  1273. break;
  1274. }
  1275. } else if (sbinfo->max_inodes) {
  1276. spin_lock(&sbinfo->stat_lock);
  1277. sbinfo->free_inodes++;
  1278. spin_unlock(&sbinfo->stat_lock);
  1279. }
  1280. return inode;
  1281. }
  1282. #ifdef CONFIG_TMPFS
  1283. static struct inode_operations shmem_symlink_inode_operations;
  1284. static struct inode_operations shmem_symlink_inline_operations;
  1285. /*
  1286. * Normally tmpfs makes no use of shmem_prepare_write, but it
  1287. * lets a tmpfs file be used read-write below the loop driver.
  1288. */
  1289. static int
  1290. shmem_prepare_write(struct file *file, struct page *page, unsigned offset, unsigned to)
  1291. {
  1292. struct inode *inode = page->mapping->host;
  1293. return shmem_getpage(inode, page->index, &page, SGP_WRITE, NULL);
  1294. }
  1295. static ssize_t
  1296. shmem_file_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
  1297. {
  1298. struct inode *inode = file->f_dentry->d_inode;
  1299. loff_t pos;
  1300. unsigned long written;
  1301. ssize_t err;
  1302. if ((ssize_t) count < 0)
  1303. return -EINVAL;
  1304. if (!access_ok(VERIFY_READ, buf, count))
  1305. return -EFAULT;
  1306. mutex_lock(&inode->i_mutex);
  1307. pos = *ppos;
  1308. written = 0;
  1309. err = generic_write_checks(file, &pos, &count, 0);
  1310. if (err || !count)
  1311. goto out;
  1312. err = remove_suid(file->f_dentry);
  1313. if (err)
  1314. goto out;
  1315. inode->i_ctime = inode->i_mtime = CURRENT_TIME;
  1316. do {
  1317. struct page *page = NULL;
  1318. unsigned long bytes, index, offset;
  1319. char *kaddr;
  1320. int left;
  1321. offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
  1322. index = pos >> PAGE_CACHE_SHIFT;
  1323. bytes = PAGE_CACHE_SIZE - offset;
  1324. if (bytes > count)
  1325. bytes = count;
  1326. /*
  1327. * We don't hold page lock across copy from user -
  1328. * what would it guard against? - so no deadlock here.
  1329. * But it still may be a good idea to prefault below.
  1330. */
  1331. err = shmem_getpage(inode, index, &page, SGP_WRITE, NULL);
  1332. if (err)
  1333. break;
  1334. left = bytes;
  1335. if (PageHighMem(page)) {
  1336. volatile unsigned char dummy;
  1337. __get_user(dummy, buf);
  1338. __get_user(dummy, buf + bytes - 1);
  1339. kaddr = kmap_atomic(page, KM_USER0);
  1340. left = __copy_from_user_inatomic(kaddr + offset,
  1341. buf, bytes);
  1342. kunmap_atomic(kaddr, KM_USER0);
  1343. }
  1344. if (left) {
  1345. kaddr = kmap(page);
  1346. left = __copy_from_user(kaddr + offset, buf, bytes);
  1347. kunmap(page);
  1348. }
  1349. written += bytes;
  1350. count -= bytes;
  1351. pos += bytes;
  1352. buf += bytes;
  1353. if (pos > inode->i_size)
  1354. i_size_write(inode, pos);
  1355. flush_dcache_page(page);
  1356. set_page_dirty(page);
  1357. mark_page_accessed(page);
  1358. page_cache_release(page);
  1359. if (left) {
  1360. pos -= left;
  1361. written -= left;
  1362. err = -EFAULT;
  1363. break;
  1364. }
  1365. /*
  1366. * Our dirty pages are not counted in nr_dirty,
  1367. * and we do not attempt to balance dirty pages.
  1368. */
  1369. cond_resched();
  1370. } while (count);
  1371. *ppos = pos;
  1372. if (written)
  1373. err = written;
  1374. out:
  1375. mutex_unlock(&inode->i_mutex);
  1376. return err;
  1377. }
  1378. static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
  1379. {
  1380. struct inode *inode = filp->f_dentry->d_inode;
  1381. struct address_space *mapping = inode->i_mapping;
  1382. unsigned long index, offset;
  1383. index = *ppos >> PAGE_CACHE_SHIFT;
  1384. offset = *ppos & ~PAGE_CACHE_MASK;
  1385. for (;;) {
  1386. struct page *page = NULL;
  1387. unsigned long end_index, nr, ret;
  1388. loff_t i_size = i_size_read(inode);
  1389. end_index = i_size >> PAGE_CACHE_SHIFT;
  1390. if (index > end_index)
  1391. break;
  1392. if (index == end_index) {
  1393. nr = i_size & ~PAGE_CACHE_MASK;
  1394. if (nr <= offset)
  1395. break;
  1396. }
  1397. desc->error = shmem_getpage(inode, index, &page, SGP_READ, NULL);
  1398. if (desc->error) {
  1399. if (desc->error == -EINVAL)
  1400. desc->error = 0;
  1401. break;
  1402. }
  1403. /*
  1404. * We must evaluate after, since reads (unlike writes)
  1405. * are called without i_mutex protection against truncate
  1406. */
  1407. nr = PAGE_CACHE_SIZE;
  1408. i_size = i_size_read(inode);
  1409. end_index = i_size >> PAGE_CACHE_SHIFT;
  1410. if (index == end_index) {
  1411. nr = i_size & ~PAGE_CACHE_MASK;
  1412. if (nr <= offset) {
  1413. if (page)
  1414. page_cache_release(page);
  1415. break;
  1416. }
  1417. }
  1418. nr -= offset;
  1419. if (page) {
  1420. /*
  1421. * If users can be writing to this page using arbitrary
  1422. * virtual addresses, take care about potential aliasing
  1423. * before reading the page on the kernel side.
  1424. */
  1425. if (mapping_writably_mapped(mapping))
  1426. flush_dcache_page(page);
  1427. /*
  1428. * Mark the page accessed if we read the beginning.
  1429. */
  1430. if (!offset)
  1431. mark_page_accessed(page);
  1432. } else {
  1433. page = ZERO_PAGE(0);
  1434. page_cache_get(page);
  1435. }
  1436. /*
  1437. * Ok, we have the page, and it's up-to-date, so
  1438. * now we can copy it to user space...
  1439. *
  1440. * The actor routine returns how many bytes were actually used..
  1441. * NOTE! This may not be the same as how much of a user buffer
  1442. * we filled up (we may be padding etc), so we can only update
  1443. * "pos" here (the actor routine has to update the user buffer
  1444. * pointers and the remaining count).
  1445. */
  1446. ret = actor(desc, page, offset, nr);
  1447. offset += ret;
  1448. index += offset >> PAGE_CACHE_SHIFT;
  1449. offset &= ~PAGE_CACHE_MASK;
  1450. page_cache_release(page);
  1451. if (ret != nr || !desc->count)
  1452. break;
  1453. cond_resched();
  1454. }
  1455. *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
  1456. file_accessed(filp);
  1457. }
  1458. static ssize_t shmem_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
  1459. {
  1460. read_descriptor_t desc;
  1461. if ((ssize_t) count < 0)
  1462. return -EINVAL;
  1463. if (!access_ok(VERIFY_WRITE, buf, count))
  1464. return -EFAULT;
  1465. if (!count)
  1466. return 0;
  1467. desc.written = 0;
  1468. desc.count = count;
  1469. desc.arg.buf = buf;
  1470. desc.error = 0;
  1471. do_shmem_file_read(filp, ppos, &desc, file_read_actor);
  1472. if (desc.written)
  1473. return desc.written;
  1474. return desc.error;
  1475. }
  1476. static ssize_t shmem_file_sendfile(struct file *in_file, loff_t *ppos,
  1477. size_t count, read_actor_t actor, void *target)
  1478. {
  1479. read_descriptor_t desc;
  1480. if (!count)
  1481. return 0;
  1482. desc.written = 0;
  1483. desc.count = count;
  1484. desc.arg.data = target;
  1485. desc.error = 0;
  1486. do_shmem_file_read(in_file, ppos, &desc, actor);
  1487. if (desc.written)
  1488. return desc.written;
  1489. return desc.error;
  1490. }
  1491. static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
  1492. {
  1493. struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
  1494. buf->f_type = TMPFS_MAGIC;
  1495. buf->f_bsize = PAGE_CACHE_SIZE;
  1496. buf->f_namelen = NAME_MAX;
  1497. spin_lock(&sbinfo->stat_lock);
  1498. if (sbinfo->max_blocks) {
  1499. buf->f_blocks = sbinfo->max_blocks;
  1500. buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
  1501. }
  1502. if (sbinfo->max_inodes) {
  1503. buf->f_files = sbinfo->max_inodes;
  1504. buf->f_ffree = sbinfo->free_inodes;
  1505. }
  1506. /* else leave those fields 0 like simple_statfs */
  1507. spin_unlock(&sbinfo->stat_lock);
  1508. return 0;
  1509. }
  1510. /*
  1511. * File creation. Allocate an inode, and we're done..
  1512. */
  1513. static int
  1514. shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
  1515. {
  1516. struct inode *inode = shmem_get_inode(dir->i_sb, mode, dev);
  1517. int error = -ENOSPC;
  1518. if (inode) {
  1519. error = security_inode_init_security(inode, dir, NULL, NULL,
  1520. NULL);
  1521. if (error) {
  1522. if (error != -EOPNOTSUPP) {
  1523. iput(inode);
  1524. return error;
  1525. }
  1526. error = 0;
  1527. }
  1528. if (dir->i_mode & S_ISGID) {
  1529. inode->i_gid = dir->i_gid;
  1530. if (S_ISDIR(mode))
  1531. inode->i_mode |= S_ISGID;
  1532. }
  1533. dir->i_size += BOGO_DIRENT_SIZE;
  1534. dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  1535. d_instantiate(dentry, inode);
  1536. dget(dentry); /* Extra count - pin the dentry in core */
  1537. }
  1538. return error;
  1539. }
  1540. static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
  1541. {
  1542. int error;
  1543. if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
  1544. return error;
  1545. dir->i_nlink++;
  1546. return 0;
  1547. }
  1548. static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
  1549. struct nameidata *nd)
  1550. {
  1551. return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
  1552. }
  1553. /*
  1554. * Link a file..
  1555. */
  1556. static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
  1557. {
  1558. struct inode *inode = old_dentry->d_inode;
  1559. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  1560. /*
  1561. * No ordinary (disk based) filesystem counts links as inodes;
  1562. * but each new link needs a new dentry, pinning lowmem, and
  1563. * tmpfs dentries cannot be pruned until they are unlinked.
  1564. */
  1565. if (sbinfo->max_inodes) {
  1566. spin_lock(&sbinfo->stat_lock);
  1567. if (!sbinfo->free_inodes) {
  1568. spin_unlock(&sbinfo->stat_lock);
  1569. return -ENOSPC;
  1570. }
  1571. sbinfo->free_inodes--;
  1572. spin_unlock(&sbinfo->stat_lock);
  1573. }
  1574. dir->i_size += BOGO_DIRENT_SIZE;
  1575. inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  1576. inode->i_nlink++;
  1577. atomic_inc(&inode->i_count); /* New dentry reference */
  1578. dget(dentry); /* Extra pinning count for the created dentry */
  1579. d_instantiate(dentry, inode);
  1580. return 0;
  1581. }
  1582. static int shmem_unlink(struct inode *dir, struct dentry *dentry)
  1583. {
  1584. struct inode *inode = dentry->d_inode;
  1585. if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) {
  1586. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  1587. if (sbinfo->max_inodes) {
  1588. spin_lock(&sbinfo->stat_lock);
  1589. sbinfo->free_inodes++;
  1590. spin_unlock(&sbinfo->stat_lock);
  1591. }
  1592. }
  1593. dir->i_size -= BOGO_DIRENT_SIZE;
  1594. inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  1595. inode->i_nlink--;
  1596. dput(dentry); /* Undo the count from "create" - this does all the work */
  1597. return 0;
  1598. }
  1599. static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
  1600. {
  1601. if (!simple_empty(dentry))
  1602. return -ENOTEMPTY;
  1603. dentry->d_inode->i_nlink--;
  1604. dir->i_nlink--;
  1605. return shmem_unlink(dir, dentry);
  1606. }
  1607. /*
  1608. * The VFS layer already does all the dentry stuff for rename,
  1609. * we just have to decrement the usage count for the target if
  1610. * it exists so that the VFS layer correctly free's it when it
  1611. * gets overwritten.
  1612. */
  1613. static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
  1614. {
  1615. struct inode *inode = old_dentry->d_inode;
  1616. int they_are_dirs = S_ISDIR(inode->i_mode);
  1617. if (!simple_empty(new_dentry))
  1618. return -ENOTEMPTY;
  1619. if (new_dentry->d_inode) {
  1620. (void) shmem_unlink(new_dir, new_dentry);
  1621. if (they_are_dirs)
  1622. old_dir->i_nlink--;
  1623. } else if (they_are_dirs) {
  1624. old_dir->i_nlink--;
  1625. new_dir->i_nlink++;
  1626. }
  1627. old_dir->i_size -= BOGO_DIRENT_SIZE;
  1628. new_dir->i_size += BOGO_DIRENT_SIZE;
  1629. old_dir->i_ctime = old_dir->i_mtime =
  1630. new_dir->i_ctime = new_dir->i_mtime =
  1631. inode->i_ctime = CURRENT_TIME;
  1632. return 0;
  1633. }
  1634. static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
  1635. {
  1636. int error;
  1637. int len;
  1638. struct inode *inode;
  1639. struct page *page = NULL;
  1640. char *kaddr;
  1641. struct shmem_inode_info *info;
  1642. len = strlen(symname) + 1;
  1643. if (len > PAGE_CACHE_SIZE)
  1644. return -ENAMETOOLONG;
  1645. inode = shmem_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0);
  1646. if (!inode)
  1647. return -ENOSPC;
  1648. error = security_inode_init_security(inode, dir, NULL, NULL,
  1649. NULL);
  1650. if (error) {
  1651. if (error != -EOPNOTSUPP) {
  1652. iput(inode);
  1653. return error;
  1654. }
  1655. error = 0;
  1656. }
  1657. info = SHMEM_I(inode);
  1658. inode->i_size = len-1;
  1659. if (len <= (char *)inode - (char *)info) {
  1660. /* do it inline */
  1661. memcpy(info, symname, len);
  1662. inode->i_op = &shmem_symlink_inline_operations;
  1663. } else {
  1664. error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
  1665. if (error) {
  1666. iput(inode);
  1667. return error;
  1668. }
  1669. inode->i_op = &shmem_symlink_inode_operations;
  1670. kaddr = kmap_atomic(page, KM_USER0);
  1671. memcpy(kaddr, symname, len);
  1672. kunmap_atomic(kaddr, KM_USER0);
  1673. set_page_dirty(page);
  1674. page_cache_release(page);
  1675. }
  1676. if (dir->i_mode & S_ISGID)
  1677. inode->i_gid = dir->i_gid;
  1678. dir->i_size += BOGO_DIRENT_SIZE;
  1679. dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  1680. d_instantiate(dentry, inode);
  1681. dget(dentry);
  1682. return 0;
  1683. }
  1684. static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
  1685. {
  1686. nd_set_link(nd, (char *)SHMEM_I(dentry->d_inode));
  1687. return NULL;
  1688. }
  1689. static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
  1690. {
  1691. struct page *page = NULL;
  1692. int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
  1693. nd_set_link(nd, res ? ERR_PTR(res) : kmap(page));
  1694. return page;
  1695. }
  1696. static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
  1697. {
  1698. if (!IS_ERR(nd_get_link(nd))) {
  1699. struct page *page = cookie;
  1700. kunmap(page);
  1701. mark_page_accessed(page);
  1702. page_cache_release(page);
  1703. }
  1704. }
  1705. static struct inode_operations shmem_symlink_inline_operations = {
  1706. .readlink = generic_readlink,
  1707. .follow_link = shmem_follow_link_inline,
  1708. };
  1709. static struct inode_operations shmem_symlink_inode_operations = {
  1710. .truncate = shmem_truncate,
  1711. .readlink = generic_readlink,
  1712. .follow_link = shmem_follow_link,
  1713. .put_link = shmem_put_link,
  1714. };
  1715. static int shmem_parse_options(char *options, int *mode, uid_t *uid,
  1716. gid_t *gid, unsigned long *blocks, unsigned long *inodes,
  1717. int *policy, nodemask_t *policy_nodes)
  1718. {
  1719. char *this_char, *value, *rest;
  1720. while (options != NULL) {
  1721. this_char = options;
  1722. for (;;) {
  1723. /*
  1724. * NUL-terminate this option: unfortunately,
  1725. * mount options form a comma-separated list,
  1726. * but mpol's nodelist may also contain commas.
  1727. */
  1728. options = strchr(options, ',');
  1729. if (options == NULL)
  1730. break;
  1731. options++;
  1732. if (!isdigit(*options)) {
  1733. options[-1] = '\0';
  1734. break;
  1735. }
  1736. }
  1737. if (!*this_char)
  1738. continue;
  1739. if ((value = strchr(this_char,'=')) != NULL) {
  1740. *value++ = 0;
  1741. } else {
  1742. printk(KERN_ERR
  1743. "tmpfs: No value for mount option '%s'\n",
  1744. this_char);
  1745. return 1;
  1746. }
  1747. if (!strcmp(this_char,"size")) {
  1748. unsigned long long size;
  1749. size = memparse(value,&rest);
  1750. if (*rest == '%') {
  1751. size <<= PAGE_SHIFT;
  1752. size *= totalram_pages;
  1753. do_div(size, 100);
  1754. rest++;
  1755. }
  1756. if (*rest)
  1757. goto bad_val;
  1758. *blocks = size >> PAGE_CACHE_SHIFT;
  1759. } else if (!strcmp(this_char,"nr_blocks")) {
  1760. *blocks = memparse(value,&rest);
  1761. if (*rest)
  1762. goto bad_val;
  1763. } else if (!strcmp(this_char,"nr_inodes")) {
  1764. *inodes = memparse(value,&rest);
  1765. if (*rest)
  1766. goto bad_val;
  1767. } else if (!strcmp(this_char,"mode")) {
  1768. if (!mode)
  1769. continue;
  1770. *mode = simple_strtoul(value,&rest,8);
  1771. if (*rest)
  1772. goto bad_val;
  1773. } else if (!strcmp(this_char,"uid")) {
  1774. if (!uid)
  1775. continue;
  1776. *uid = simple_strtoul(value,&rest,0);
  1777. if (*rest)
  1778. goto bad_val;
  1779. } else if (!strcmp(this_char,"gid")) {
  1780. if (!gid)
  1781. continue;
  1782. *gid = simple_strtoul(value,&rest,0);
  1783. if (*rest)
  1784. goto bad_val;
  1785. } else if (!strcmp(this_char,"mpol")) {
  1786. if (shmem_parse_mpol(value,policy,policy_nodes))
  1787. goto bad_val;
  1788. } else {
  1789. printk(KERN_ERR "tmpfs: Bad mount option %s\n",
  1790. this_char);
  1791. return 1;
  1792. }
  1793. }
  1794. return 0;
  1795. bad_val:
  1796. printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
  1797. value, this_char);
  1798. return 1;
  1799. }
  1800. static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
  1801. {
  1802. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  1803. unsigned long max_blocks = sbinfo->max_blocks;
  1804. unsigned long max_inodes = sbinfo->max_inodes;
  1805. int policy = sbinfo->policy;
  1806. nodemask_t policy_nodes = sbinfo->policy_nodes;
  1807. unsigned long blocks;
  1808. unsigned long inodes;
  1809. int error = -EINVAL;
  1810. if (shmem_parse_options(data, NULL, NULL, NULL, &max_blocks,
  1811. &max_inodes, &policy, &policy_nodes))
  1812. return error;
  1813. spin_lock(&sbinfo->stat_lock);
  1814. blocks = sbinfo->max_blocks - sbinfo->free_blocks;
  1815. inodes = sbinfo->max_inodes - sbinfo->free_inodes;
  1816. if (max_blocks < blocks)
  1817. goto out;
  1818. if (max_inodes < inodes)
  1819. goto out;
  1820. /*
  1821. * Those tests also disallow limited->unlimited while any are in
  1822. * use, so i_blocks will always be zero when max_blocks is zero;
  1823. * but we must separately disallow unlimited->limited, because
  1824. * in that case we have no record of how much is already in use.
  1825. */
  1826. if (max_blocks && !sbinfo->max_blocks)
  1827. goto out;
  1828. if (max_inodes && !sbinfo->max_inodes)
  1829. goto out;
  1830. error = 0;
  1831. sbinfo->max_blocks = max_blocks;
  1832. sbinfo->free_blocks = max_blocks - blocks;
  1833. sbinfo->max_inodes = max_inodes;
  1834. sbinfo->free_inodes = max_inodes - inodes;
  1835. sbinfo->policy = policy;
  1836. sbinfo->policy_nodes = policy_nodes;
  1837. out:
  1838. spin_unlock(&sbinfo->stat_lock);
  1839. return error;
  1840. }
  1841. #endif
  1842. static void shmem_put_super(struct super_block *sb)
  1843. {
  1844. kfree(sb->s_fs_info);
  1845. sb->s_fs_info = NULL;
  1846. }
  1847. static int shmem_fill_super(struct super_block *sb,
  1848. void *data, int silent)
  1849. {
  1850. struct inode *inode;
  1851. struct dentry *root;
  1852. int mode = S_IRWXUGO | S_ISVTX;
  1853. uid_t uid = current->fsuid;
  1854. gid_t gid = current->fsgid;
  1855. int err = -ENOMEM;
  1856. struct shmem_sb_info *sbinfo;
  1857. unsigned long blocks = 0;
  1858. unsigned long inodes = 0;
  1859. int policy = MPOL_DEFAULT;
  1860. nodemask_t policy_nodes = node_online_map;
  1861. #ifdef CONFIG_TMPFS
  1862. /*
  1863. * Per default we only allow half of the physical ram per
  1864. * tmpfs instance, limiting inodes to one per page of lowmem;
  1865. * but the internal instance is left unlimited.
  1866. */
  1867. if (!(sb->s_flags & MS_NOUSER)) {
  1868. blocks = totalram_pages / 2;
  1869. inodes = totalram_pages - totalhigh_pages;
  1870. if (inodes > blocks)
  1871. inodes = blocks;
  1872. if (shmem_parse_options(data, &mode, &uid, &gid, &blocks,
  1873. &inodes, &policy, &policy_nodes))
  1874. return -EINVAL;
  1875. }
  1876. #else
  1877. sb->s_flags |= MS_NOUSER;
  1878. #endif
  1879. /* Round up to L1_CACHE_BYTES to resist false sharing */
  1880. sbinfo = kmalloc(max((int)sizeof(struct shmem_sb_info),
  1881. L1_CACHE_BYTES), GFP_KERNEL);
  1882. if (!sbinfo)
  1883. return -ENOMEM;
  1884. spin_lock_init(&sbinfo->stat_lock);
  1885. sbinfo->max_blocks = blocks;
  1886. sbinfo->free_blocks = blocks;
  1887. sbinfo->max_inodes = inodes;
  1888. sbinfo->free_inodes = inodes;
  1889. sbinfo->policy = policy;
  1890. sbinfo->policy_nodes = policy_nodes;
  1891. sb->s_fs_info = sbinfo;
  1892. sb->s_maxbytes = SHMEM_MAX_BYTES;
  1893. sb->s_blocksize = PAGE_CACHE_SIZE;
  1894. sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
  1895. sb->s_magic = TMPFS_MAGIC;
  1896. sb->s_op = &shmem_ops;
  1897. sb->s_time_gran = 1;
  1898. inode = shmem_get_inode(sb, S_IFDIR | mode, 0);
  1899. if (!inode)
  1900. goto failed;
  1901. inode->i_uid = uid;
  1902. inode->i_gid = gid;
  1903. root = d_alloc_root(inode);
  1904. if (!root)
  1905. goto failed_iput;
  1906. sb->s_root = root;
  1907. return 0;
  1908. failed_iput:
  1909. iput(inode);
  1910. failed:
  1911. shmem_put_super(sb);
  1912. return err;
  1913. }
  1914. static struct kmem_cache *shmem_inode_cachep;
  1915. static struct inode *shmem_alloc_inode(struct super_block *sb)
  1916. {
  1917. struct shmem_inode_info *p;
  1918. p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, SLAB_KERNEL);
  1919. if (!p)
  1920. return NULL;
  1921. return &p->vfs_inode;
  1922. }
  1923. static void shmem_destroy_inode(struct inode *inode)
  1924. {
  1925. if ((inode->i_mode & S_IFMT) == S_IFREG) {
  1926. /* only struct inode is valid if it's an inline symlink */
  1927. mpol_free_shared_policy(&SHMEM_I(inode)->policy);
  1928. }
  1929. kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
  1930. }
  1931. static void init_once(void *foo, struct kmem_cache *cachep,
  1932. unsigned long flags)
  1933. {
  1934. struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
  1935. if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
  1936. SLAB_CTOR_CONSTRUCTOR) {
  1937. inode_init_once(&p->vfs_inode);
  1938. }
  1939. }
  1940. static int init_inodecache(void)
  1941. {
  1942. shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
  1943. sizeof(struct shmem_inode_info),
  1944. 0, 0, init_once, NULL);
  1945. if (shmem_inode_cachep == NULL)
  1946. return -ENOMEM;
  1947. return 0;
  1948. }
  1949. static void destroy_inodecache(void)
  1950. {
  1951. if (kmem_cache_destroy(shmem_inode_cachep))
  1952. printk(KERN_INFO "shmem_inode_cache: not all structures were freed\n");
  1953. }
  1954. static struct address_space_operations shmem_aops = {
  1955. .writepage = shmem_writepage,
  1956. .set_page_dirty = __set_page_dirty_nobuffers,
  1957. #ifdef CONFIG_TMPFS
  1958. .prepare_write = shmem_prepare_write,
  1959. .commit_write = simple_commit_write,
  1960. #endif
  1961. .migratepage = migrate_page,
  1962. };
  1963. static struct file_operations shmem_file_operations = {
  1964. .mmap = shmem_mmap,
  1965. #ifdef CONFIG_TMPFS
  1966. .llseek = generic_file_llseek,
  1967. .read = shmem_file_read,
  1968. .write = shmem_file_write,
  1969. .fsync = simple_sync_file,
  1970. .sendfile = shmem_file_sendfile,
  1971. #endif
  1972. };
  1973. static struct inode_operations shmem_inode_operations = {
  1974. .truncate = shmem_truncate,
  1975. .setattr = shmem_notify_change,
  1976. .truncate_range = shmem_truncate_range,
  1977. };
  1978. static struct inode_operations shmem_dir_inode_operations = {
  1979. #ifdef CONFIG_TMPFS
  1980. .create = shmem_create,
  1981. .lookup = simple_lookup,
  1982. .link = shmem_link,
  1983. .unlink = shmem_unlink,
  1984. .symlink = shmem_symlink,
  1985. .mkdir = shmem_mkdir,
  1986. .rmdir = shmem_rmdir,
  1987. .mknod = shmem_mknod,
  1988. .rename = shmem_rename,
  1989. #endif
  1990. };
  1991. static struct super_operations shmem_ops = {
  1992. .alloc_inode = shmem_alloc_inode,
  1993. .destroy_inode = shmem_destroy_inode,
  1994. #ifdef CONFIG_TMPFS
  1995. .statfs = shmem_statfs,
  1996. .remount_fs = shmem_remount_fs,
  1997. #endif
  1998. .delete_inode = shmem_delete_inode,
  1999. .drop_inode = generic_delete_inode,
  2000. .put_super = shmem_put_super,
  2001. };
  2002. static struct vm_operations_struct shmem_vm_ops = {
  2003. .nopage = shmem_nopage,
  2004. .populate = shmem_populate,
  2005. #ifdef CONFIG_NUMA
  2006. .set_policy = shmem_set_policy,
  2007. .get_policy = shmem_get_policy,
  2008. #endif
  2009. };
  2010. static int shmem_get_sb(struct file_system_type *fs_type,
  2011. int flags, const char *dev_name, void *data, struct vfsmount *mnt)
  2012. {
  2013. return get_sb_nodev(fs_type, flags, data, shmem_fill_super, mnt);
  2014. }
  2015. static struct file_system_type tmpfs_fs_type = {
  2016. .owner = THIS_MODULE,
  2017. .name = "tmpfs",
  2018. .get_sb = shmem_get_sb,
  2019. .kill_sb = kill_litter_super,
  2020. };
  2021. static struct vfsmount *shm_mnt;
  2022. static int __init init_tmpfs(void)
  2023. {
  2024. int error;
  2025. error = init_inodecache();
  2026. if (error)
  2027. goto out3;
  2028. error = register_filesystem(&tmpfs_fs_type);
  2029. if (error) {
  2030. printk(KERN_ERR "Could not register tmpfs\n");
  2031. goto out2;
  2032. }
  2033. #ifdef CONFIG_TMPFS
  2034. devfs_mk_dir("shm");
  2035. #endif
  2036. shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER,
  2037. tmpfs_fs_type.name, NULL);
  2038. if (IS_ERR(shm_mnt)) {
  2039. error = PTR_ERR(shm_mnt);
  2040. printk(KERN_ERR "Could not kern_mount tmpfs\n");
  2041. goto out1;
  2042. }
  2043. return 0;
  2044. out1:
  2045. unregister_filesystem(&tmpfs_fs_type);
  2046. out2:
  2047. destroy_inodecache();
  2048. out3:
  2049. shm_mnt = ERR_PTR(error);
  2050. return error;
  2051. }
  2052. module_init(init_tmpfs)
  2053. /*
  2054. * shmem_file_setup - get an unlinked file living in tmpfs
  2055. *
  2056. * @name: name for dentry (to be seen in /proc/<pid>/maps
  2057. * @size: size to be set for the file
  2058. *
  2059. */
  2060. struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
  2061. {
  2062. int error;
  2063. struct file *file;
  2064. struct inode *inode;
  2065. struct dentry *dentry, *root;
  2066. struct qstr this;
  2067. if (IS_ERR(shm_mnt))
  2068. return (void *)shm_mnt;
  2069. if (size < 0 || size > SHMEM_MAX_BYTES)
  2070. return ERR_PTR(-EINVAL);
  2071. if (shmem_acct_size(flags, size))
  2072. return ERR_PTR(-ENOMEM);
  2073. error = -ENOMEM;
  2074. this.name = name;
  2075. this.len = strlen(name);
  2076. this.hash = 0; /* will go */
  2077. root = shm_mnt->mnt_root;
  2078. dentry = d_alloc(root, &this);
  2079. if (!dentry)
  2080. goto put_memory;
  2081. error = -ENFILE;
  2082. file = get_empty_filp();
  2083. if (!file)
  2084. goto put_dentry;
  2085. error = -ENOSPC;
  2086. inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0);
  2087. if (!inode)
  2088. goto close_file;
  2089. SHMEM_I(inode)->flags = flags & VM_ACCOUNT;
  2090. d_instantiate(dentry, inode);
  2091. inode->i_size = size;
  2092. inode->i_nlink = 0; /* It is unlinked */
  2093. file->f_vfsmnt = mntget(shm_mnt);
  2094. file->f_dentry = dentry;
  2095. file->f_mapping = inode->i_mapping;
  2096. file->f_op = &shmem_file_operations;
  2097. file->f_mode = FMODE_WRITE | FMODE_READ;
  2098. return file;
  2099. close_file:
  2100. put_filp(file);
  2101. put_dentry:
  2102. dput(dentry);
  2103. put_memory:
  2104. shmem_unacct_size(flags, size);
  2105. return ERR_PTR(error);
  2106. }
  2107. /*
  2108. * shmem_zero_setup - setup a shared anonymous mapping
  2109. *
  2110. * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
  2111. */
  2112. int shmem_zero_setup(struct vm_area_struct *vma)
  2113. {
  2114. struct file *file;
  2115. loff_t size = vma->vm_end - vma->vm_start;
  2116. file = shmem_file_setup("dev/zero", size, vma->vm_flags);
  2117. if (IS_ERR(file))
  2118. return PTR_ERR(file);
  2119. if (vma->vm_file)
  2120. fput(vma->vm_file);
  2121. vma->vm_file = file;
  2122. vma->vm_ops = &shmem_vm_ops;
  2123. return 0;
  2124. }