shmem.c 73 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817
  1. /*
  2. * Resizable virtual memory filesystem for Linux.
  3. *
  4. * Copyright (C) 2000 Linus Torvalds.
  5. * 2000 Transmeta Corp.
  6. * 2000-2001 Christoph Rohland
  7. * 2000-2001 SAP AG
  8. * 2002 Red Hat Inc.
  9. * Copyright (C) 2002-2005 Hugh Dickins.
  10. * Copyright (C) 2002-2005 VERITAS Software Corporation.
  11. * Copyright (C) 2004 Andi Kleen, SuSE Labs
  12. *
  13. * Extended attribute support for tmpfs:
  14. * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
  15. * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
  16. *
  17. * tiny-shmem:
  18. * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
  19. *
  20. * This file is released under the GPL.
  21. */
  22. #include <linux/fs.h>
  23. #include <linux/init.h>
  24. #include <linux/vfs.h>
  25. #include <linux/mount.h>
  26. #include <linux/pagemap.h>
  27. #include <linux/file.h>
  28. #include <linux/mm.h>
  29. #include <linux/module.h>
  30. #include <linux/percpu_counter.h>
  31. #include <linux/swap.h>
  32. static struct vfsmount *shm_mnt;
  33. #ifdef CONFIG_SHMEM
  34. /*
  35. * This virtual memory filesystem is heavily based on the ramfs. It
  36. * extends ramfs by the ability to use swap and honor resource limits
  37. * which makes it a completely usable filesystem.
  38. */
  39. #include <linux/xattr.h>
  40. #include <linux/exportfs.h>
  41. #include <linux/posix_acl.h>
  42. #include <linux/generic_acl.h>
  43. #include <linux/mman.h>
  44. #include <linux/string.h>
  45. #include <linux/slab.h>
  46. #include <linux/backing-dev.h>
  47. #include <linux/shmem_fs.h>
  48. #include <linux/writeback.h>
  49. #include <linux/blkdev.h>
  50. #include <linux/security.h>
  51. #include <linux/swapops.h>
  52. #include <linux/mempolicy.h>
  53. #include <linux/namei.h>
  54. #include <linux/ctype.h>
  55. #include <linux/migrate.h>
  56. #include <linux/highmem.h>
  57. #include <linux/seq_file.h>
  58. #include <linux/magic.h>
  59. #include <asm/uaccess.h>
  60. #include <asm/div64.h>
  61. #include <asm/pgtable.h>
  62. /*
  63. * The maximum size of a shmem/tmpfs file is limited by the maximum size of
  64. * its triple-indirect swap vector - see illustration at shmem_swp_entry().
  65. *
  66. * With 4kB page size, maximum file size is just over 2TB on a 32-bit kernel,
  67. * but one eighth of that on a 64-bit kernel. With 8kB page size, maximum
  68. * file size is just over 4TB on a 64-bit kernel, but 16TB on a 32-bit kernel,
  69. * MAX_LFS_FILESIZE being then more restrictive than swap vector layout.
  70. *
  71. * We use / and * instead of shifts in the definitions below, so that the swap
  72. * vector can be tested with small even values (e.g. 20) for ENTRIES_PER_PAGE.
  73. */
  74. #define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
  75. #define ENTRIES_PER_PAGEPAGE ((unsigned long long)ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
  76. #define SHMSWP_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
  77. #define SHMSWP_MAX_BYTES (SHMSWP_MAX_INDEX << PAGE_CACHE_SHIFT)
  78. #define SHMEM_MAX_BYTES min_t(unsigned long long, SHMSWP_MAX_BYTES, MAX_LFS_FILESIZE)
  79. #define SHMEM_MAX_INDEX ((unsigned long)((SHMEM_MAX_BYTES+1) >> PAGE_CACHE_SHIFT))
  80. #define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
  81. #define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
  82. /* info->flags needs VM_flags to handle pagein/truncate races efficiently */
  83. #define SHMEM_PAGEIN VM_READ
  84. #define SHMEM_TRUNCATE VM_WRITE
  85. /* Definition to limit shmem_truncate's steps between cond_rescheds */
  86. #define LATENCY_LIMIT 64
  87. /* Pretend that each entry is of this size in directory's i_size */
  88. #define BOGO_DIRENT_SIZE 20
  89. /* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */
  90. enum sgp_type {
  91. SGP_READ, /* don't exceed i_size, don't allocate page */
  92. SGP_CACHE, /* don't exceed i_size, may allocate page */
  93. SGP_DIRTY, /* like SGP_CACHE, but set new page dirty */
  94. SGP_WRITE, /* may exceed i_size, may allocate page */
  95. };
  96. #ifdef CONFIG_TMPFS
  97. static unsigned long shmem_default_max_blocks(void)
  98. {
  99. return totalram_pages / 2;
  100. }
  101. static unsigned long shmem_default_max_inodes(void)
  102. {
  103. return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
  104. }
  105. #endif
  106. static int shmem_getpage(struct inode *inode, unsigned long idx,
  107. struct page **pagep, enum sgp_type sgp, int *type);
  108. static inline struct page *shmem_dir_alloc(gfp_t gfp_mask)
  109. {
  110. /*
  111. * The above definition of ENTRIES_PER_PAGE, and the use of
  112. * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE:
  113. * might be reconsidered if it ever diverges from PAGE_SIZE.
  114. *
  115. * Mobility flags are masked out as swap vectors cannot move
  116. */
  117. return alloc_pages((gfp_mask & ~GFP_MOVABLE_MASK) | __GFP_ZERO,
  118. PAGE_CACHE_SHIFT-PAGE_SHIFT);
  119. }
  120. static inline void shmem_dir_free(struct page *page)
  121. {
  122. __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT);
  123. }
  124. static struct page **shmem_dir_map(struct page *page)
  125. {
  126. return (struct page **)kmap_atomic(page, KM_USER0);
  127. }
  128. static inline void shmem_dir_unmap(struct page **dir)
  129. {
  130. kunmap_atomic(dir, KM_USER0);
  131. }
  132. static swp_entry_t *shmem_swp_map(struct page *page)
  133. {
  134. return (swp_entry_t *)kmap_atomic(page, KM_USER1);
  135. }
  136. static inline void shmem_swp_balance_unmap(void)
  137. {
  138. /*
  139. * When passing a pointer to an i_direct entry, to code which
  140. * also handles indirect entries and so will shmem_swp_unmap,
  141. * we must arrange for the preempt count to remain in balance.
  142. * What kmap_atomic of a lowmem page does depends on config
  143. * and architecture, so pretend to kmap_atomic some lowmem page.
  144. */
  145. (void) kmap_atomic(ZERO_PAGE(0), KM_USER1);
  146. }
  147. static inline void shmem_swp_unmap(swp_entry_t *entry)
  148. {
  149. kunmap_atomic(entry, KM_USER1);
  150. }
  151. static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
  152. {
  153. return sb->s_fs_info;
  154. }
  155. /*
  156. * shmem_file_setup pre-accounts the whole fixed size of a VM object,
  157. * for shared memory and for shared anonymous (/dev/zero) mappings
  158. * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
  159. * consistent with the pre-accounting of private mappings ...
  160. */
  161. static inline int shmem_acct_size(unsigned long flags, loff_t size)
  162. {
  163. return (flags & VM_NORESERVE) ?
  164. 0 : security_vm_enough_memory_kern(VM_ACCT(size));
  165. }
  166. static inline void shmem_unacct_size(unsigned long flags, loff_t size)
  167. {
  168. if (!(flags & VM_NORESERVE))
  169. vm_unacct_memory(VM_ACCT(size));
  170. }
  171. /*
  172. * ... whereas tmpfs objects are accounted incrementally as
  173. * pages are allocated, in order to allow huge sparse files.
  174. * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
  175. * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
  176. */
  177. static inline int shmem_acct_block(unsigned long flags)
  178. {
  179. return (flags & VM_NORESERVE) ?
  180. security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE)) : 0;
  181. }
  182. static inline void shmem_unacct_blocks(unsigned long flags, long pages)
  183. {
  184. if (flags & VM_NORESERVE)
  185. vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
  186. }
  187. static const struct super_operations shmem_ops;
  188. static const struct address_space_operations shmem_aops;
  189. static const struct file_operations shmem_file_operations;
  190. static const struct inode_operations shmem_inode_operations;
  191. static const struct inode_operations shmem_dir_inode_operations;
  192. static const struct inode_operations shmem_special_inode_operations;
  193. static const struct vm_operations_struct shmem_vm_ops;
  194. static struct backing_dev_info shmem_backing_dev_info __read_mostly = {
  195. .ra_pages = 0, /* No readahead */
  196. .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
  197. };
  198. static LIST_HEAD(shmem_swaplist);
  199. static DEFINE_MUTEX(shmem_swaplist_mutex);
  200. static void shmem_free_blocks(struct inode *inode, long pages)
  201. {
  202. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  203. if (sbinfo->max_blocks) {
  204. percpu_counter_add(&sbinfo->used_blocks, -pages);
  205. spin_lock(&inode->i_lock);
  206. inode->i_blocks -= pages*BLOCKS_PER_PAGE;
  207. spin_unlock(&inode->i_lock);
  208. }
  209. }
  210. static int shmem_reserve_inode(struct super_block *sb)
  211. {
  212. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  213. if (sbinfo->max_inodes) {
  214. spin_lock(&sbinfo->stat_lock);
  215. if (!sbinfo->free_inodes) {
  216. spin_unlock(&sbinfo->stat_lock);
  217. return -ENOSPC;
  218. }
  219. sbinfo->free_inodes--;
  220. spin_unlock(&sbinfo->stat_lock);
  221. }
  222. return 0;
  223. }
  224. static void shmem_free_inode(struct super_block *sb)
  225. {
  226. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  227. if (sbinfo->max_inodes) {
  228. spin_lock(&sbinfo->stat_lock);
  229. sbinfo->free_inodes++;
  230. spin_unlock(&sbinfo->stat_lock);
  231. }
  232. }
  233. /**
  234. * shmem_recalc_inode - recalculate the size of an inode
  235. * @inode: inode to recalc
  236. *
  237. * We have to calculate the free blocks since the mm can drop
  238. * undirtied hole pages behind our back.
  239. *
  240. * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
  241. * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
  242. *
  243. * It has to be called with the spinlock held.
  244. */
  245. static void shmem_recalc_inode(struct inode *inode)
  246. {
  247. struct shmem_inode_info *info = SHMEM_I(inode);
  248. long freed;
  249. freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
  250. if (freed > 0) {
  251. info->alloced -= freed;
  252. shmem_unacct_blocks(info->flags, freed);
  253. shmem_free_blocks(inode, freed);
  254. }
  255. }
  256. /**
  257. * shmem_swp_entry - find the swap vector position in the info structure
  258. * @info: info structure for the inode
  259. * @index: index of the page to find
  260. * @page: optional page to add to the structure. Has to be preset to
  261. * all zeros
  262. *
  263. * If there is no space allocated yet it will return NULL when
  264. * page is NULL, else it will use the page for the needed block,
  265. * setting it to NULL on return to indicate that it has been used.
  266. *
  267. * The swap vector is organized the following way:
  268. *
  269. * There are SHMEM_NR_DIRECT entries directly stored in the
  270. * shmem_inode_info structure. So small files do not need an addional
  271. * allocation.
  272. *
  273. * For pages with index > SHMEM_NR_DIRECT there is the pointer
  274. * i_indirect which points to a page which holds in the first half
  275. * doubly indirect blocks, in the second half triple indirect blocks:
  276. *
  277. * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the
  278. * following layout (for SHMEM_NR_DIRECT == 16):
  279. *
  280. * i_indirect -> dir --> 16-19
  281. * | +-> 20-23
  282. * |
  283. * +-->dir2 --> 24-27
  284. * | +-> 28-31
  285. * | +-> 32-35
  286. * | +-> 36-39
  287. * |
  288. * +-->dir3 --> 40-43
  289. * +-> 44-47
  290. * +-> 48-51
  291. * +-> 52-55
  292. */
  293. static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page)
  294. {
  295. unsigned long offset;
  296. struct page **dir;
  297. struct page *subdir;
  298. if (index < SHMEM_NR_DIRECT) {
  299. shmem_swp_balance_unmap();
  300. return info->i_direct+index;
  301. }
  302. if (!info->i_indirect) {
  303. if (page) {
  304. info->i_indirect = *page;
  305. *page = NULL;
  306. }
  307. return NULL; /* need another page */
  308. }
  309. index -= SHMEM_NR_DIRECT;
  310. offset = index % ENTRIES_PER_PAGE;
  311. index /= ENTRIES_PER_PAGE;
  312. dir = shmem_dir_map(info->i_indirect);
  313. if (index >= ENTRIES_PER_PAGE/2) {
  314. index -= ENTRIES_PER_PAGE/2;
  315. dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE;
  316. index %= ENTRIES_PER_PAGE;
  317. subdir = *dir;
  318. if (!subdir) {
  319. if (page) {
  320. *dir = *page;
  321. *page = NULL;
  322. }
  323. shmem_dir_unmap(dir);
  324. return NULL; /* need another page */
  325. }
  326. shmem_dir_unmap(dir);
  327. dir = shmem_dir_map(subdir);
  328. }
  329. dir += index;
  330. subdir = *dir;
  331. if (!subdir) {
  332. if (!page || !(subdir = *page)) {
  333. shmem_dir_unmap(dir);
  334. return NULL; /* need a page */
  335. }
  336. *dir = subdir;
  337. *page = NULL;
  338. }
  339. shmem_dir_unmap(dir);
  340. return shmem_swp_map(subdir) + offset;
  341. }
  342. static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value)
  343. {
  344. long incdec = value? 1: -1;
  345. entry->val = value;
  346. info->swapped += incdec;
  347. if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) {
  348. struct page *page = kmap_atomic_to_page(entry);
  349. set_page_private(page, page_private(page) + incdec);
  350. }
  351. }
  352. /**
  353. * shmem_swp_alloc - get the position of the swap entry for the page.
  354. * @info: info structure for the inode
  355. * @index: index of the page to find
  356. * @sgp: check and recheck i_size? skip allocation?
  357. *
  358. * If the entry does not exist, allocate it.
  359. */
  360. static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp)
  361. {
  362. struct inode *inode = &info->vfs_inode;
  363. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  364. struct page *page = NULL;
  365. swp_entry_t *entry;
  366. if (sgp != SGP_WRITE &&
  367. ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode))
  368. return ERR_PTR(-EINVAL);
  369. while (!(entry = shmem_swp_entry(info, index, &page))) {
  370. if (sgp == SGP_READ)
  371. return shmem_swp_map(ZERO_PAGE(0));
  372. /*
  373. * Test used_blocks against 1 less max_blocks, since we have 1 data
  374. * page (and perhaps indirect index pages) yet to allocate:
  375. * a waste to allocate index if we cannot allocate data.
  376. */
  377. if (sbinfo->max_blocks) {
  378. if (percpu_counter_compare(&sbinfo->used_blocks,
  379. sbinfo->max_blocks - 1) >= 0)
  380. return ERR_PTR(-ENOSPC);
  381. percpu_counter_inc(&sbinfo->used_blocks);
  382. spin_lock(&inode->i_lock);
  383. inode->i_blocks += BLOCKS_PER_PAGE;
  384. spin_unlock(&inode->i_lock);
  385. }
  386. spin_unlock(&info->lock);
  387. page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping));
  388. spin_lock(&info->lock);
  389. if (!page) {
  390. shmem_free_blocks(inode, 1);
  391. return ERR_PTR(-ENOMEM);
  392. }
  393. if (sgp != SGP_WRITE &&
  394. ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
  395. entry = ERR_PTR(-EINVAL);
  396. break;
  397. }
  398. if (info->next_index <= index)
  399. info->next_index = index + 1;
  400. }
  401. if (page) {
  402. /* another task gave its page, or truncated the file */
  403. shmem_free_blocks(inode, 1);
  404. shmem_dir_free(page);
  405. }
  406. if (info->next_index <= index && !IS_ERR(entry))
  407. info->next_index = index + 1;
  408. return entry;
  409. }
  410. /**
  411. * shmem_free_swp - free some swap entries in a directory
  412. * @dir: pointer to the directory
  413. * @edir: pointer after last entry of the directory
  414. * @punch_lock: pointer to spinlock when needed for the holepunch case
  415. */
  416. static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir,
  417. spinlock_t *punch_lock)
  418. {
  419. spinlock_t *punch_unlock = NULL;
  420. swp_entry_t *ptr;
  421. int freed = 0;
  422. for (ptr = dir; ptr < edir; ptr++) {
  423. if (ptr->val) {
  424. if (unlikely(punch_lock)) {
  425. punch_unlock = punch_lock;
  426. punch_lock = NULL;
  427. spin_lock(punch_unlock);
  428. if (!ptr->val)
  429. continue;
  430. }
  431. free_swap_and_cache(*ptr);
  432. *ptr = (swp_entry_t){0};
  433. freed++;
  434. }
  435. }
  436. if (punch_unlock)
  437. spin_unlock(punch_unlock);
  438. return freed;
  439. }
  440. static int shmem_map_and_free_swp(struct page *subdir, int offset,
  441. int limit, struct page ***dir, spinlock_t *punch_lock)
  442. {
  443. swp_entry_t *ptr;
  444. int freed = 0;
  445. ptr = shmem_swp_map(subdir);
  446. for (; offset < limit; offset += LATENCY_LIMIT) {
  447. int size = limit - offset;
  448. if (size > LATENCY_LIMIT)
  449. size = LATENCY_LIMIT;
  450. freed += shmem_free_swp(ptr+offset, ptr+offset+size,
  451. punch_lock);
  452. if (need_resched()) {
  453. shmem_swp_unmap(ptr);
  454. if (*dir) {
  455. shmem_dir_unmap(*dir);
  456. *dir = NULL;
  457. }
  458. cond_resched();
  459. ptr = shmem_swp_map(subdir);
  460. }
  461. }
  462. shmem_swp_unmap(ptr);
  463. return freed;
  464. }
  465. static void shmem_free_pages(struct list_head *next)
  466. {
  467. struct page *page;
  468. int freed = 0;
  469. do {
  470. page = container_of(next, struct page, lru);
  471. next = next->next;
  472. shmem_dir_free(page);
  473. freed++;
  474. if (freed >= LATENCY_LIMIT) {
  475. cond_resched();
  476. freed = 0;
  477. }
  478. } while (next);
  479. }
  480. static void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end)
  481. {
  482. struct shmem_inode_info *info = SHMEM_I(inode);
  483. unsigned long idx;
  484. unsigned long size;
  485. unsigned long limit;
  486. unsigned long stage;
  487. unsigned long diroff;
  488. struct page **dir;
  489. struct page *topdir;
  490. struct page *middir;
  491. struct page *subdir;
  492. swp_entry_t *ptr;
  493. LIST_HEAD(pages_to_free);
  494. long nr_pages_to_free = 0;
  495. long nr_swaps_freed = 0;
  496. int offset;
  497. int freed;
  498. int punch_hole;
  499. spinlock_t *needs_lock;
  500. spinlock_t *punch_lock;
  501. unsigned long upper_limit;
  502. inode->i_ctime = inode->i_mtime = CURRENT_TIME;
  503. idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  504. if (idx >= info->next_index)
  505. return;
  506. spin_lock(&info->lock);
  507. info->flags |= SHMEM_TRUNCATE;
  508. if (likely(end == (loff_t) -1)) {
  509. limit = info->next_index;
  510. upper_limit = SHMEM_MAX_INDEX;
  511. info->next_index = idx;
  512. needs_lock = NULL;
  513. punch_hole = 0;
  514. } else {
  515. if (end + 1 >= inode->i_size) { /* we may free a little more */
  516. limit = (inode->i_size + PAGE_CACHE_SIZE - 1) >>
  517. PAGE_CACHE_SHIFT;
  518. upper_limit = SHMEM_MAX_INDEX;
  519. } else {
  520. limit = (end + 1) >> PAGE_CACHE_SHIFT;
  521. upper_limit = limit;
  522. }
  523. needs_lock = &info->lock;
  524. punch_hole = 1;
  525. }
  526. topdir = info->i_indirect;
  527. if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) {
  528. info->i_indirect = NULL;
  529. nr_pages_to_free++;
  530. list_add(&topdir->lru, &pages_to_free);
  531. }
  532. spin_unlock(&info->lock);
  533. if (info->swapped && idx < SHMEM_NR_DIRECT) {
  534. ptr = info->i_direct;
  535. size = limit;
  536. if (size > SHMEM_NR_DIRECT)
  537. size = SHMEM_NR_DIRECT;
  538. nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size, needs_lock);
  539. }
  540. /*
  541. * If there are no indirect blocks or we are punching a hole
  542. * below indirect blocks, nothing to be done.
  543. */
  544. if (!topdir || limit <= SHMEM_NR_DIRECT)
  545. goto done2;
  546. /*
  547. * The truncation case has already dropped info->lock, and we're safe
  548. * because i_size and next_index have already been lowered, preventing
  549. * access beyond. But in the punch_hole case, we still need to take
  550. * the lock when updating the swap directory, because there might be
  551. * racing accesses by shmem_getpage(SGP_CACHE), shmem_unuse_inode or
  552. * shmem_writepage. However, whenever we find we can remove a whole
  553. * directory page (not at the misaligned start or end of the range),
  554. * we first NULLify its pointer in the level above, and then have no
  555. * need to take the lock when updating its contents: needs_lock and
  556. * punch_lock (either pointing to info->lock or NULL) manage this.
  557. */
  558. upper_limit -= SHMEM_NR_DIRECT;
  559. limit -= SHMEM_NR_DIRECT;
  560. idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0;
  561. offset = idx % ENTRIES_PER_PAGE;
  562. idx -= offset;
  563. dir = shmem_dir_map(topdir);
  564. stage = ENTRIES_PER_PAGEPAGE/2;
  565. if (idx < ENTRIES_PER_PAGEPAGE/2) {
  566. middir = topdir;
  567. diroff = idx/ENTRIES_PER_PAGE;
  568. } else {
  569. dir += ENTRIES_PER_PAGE/2;
  570. dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE;
  571. while (stage <= idx)
  572. stage += ENTRIES_PER_PAGEPAGE;
  573. middir = *dir;
  574. if (*dir) {
  575. diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) %
  576. ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE;
  577. if (!diroff && !offset && upper_limit >= stage) {
  578. if (needs_lock) {
  579. spin_lock(needs_lock);
  580. *dir = NULL;
  581. spin_unlock(needs_lock);
  582. needs_lock = NULL;
  583. } else
  584. *dir = NULL;
  585. nr_pages_to_free++;
  586. list_add(&middir->lru, &pages_to_free);
  587. }
  588. shmem_dir_unmap(dir);
  589. dir = shmem_dir_map(middir);
  590. } else {
  591. diroff = 0;
  592. offset = 0;
  593. idx = stage;
  594. }
  595. }
  596. for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) {
  597. if (unlikely(idx == stage)) {
  598. shmem_dir_unmap(dir);
  599. dir = shmem_dir_map(topdir) +
  600. ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
  601. while (!*dir) {
  602. dir++;
  603. idx += ENTRIES_PER_PAGEPAGE;
  604. if (idx >= limit)
  605. goto done1;
  606. }
  607. stage = idx + ENTRIES_PER_PAGEPAGE;
  608. middir = *dir;
  609. if (punch_hole)
  610. needs_lock = &info->lock;
  611. if (upper_limit >= stage) {
  612. if (needs_lock) {
  613. spin_lock(needs_lock);
  614. *dir = NULL;
  615. spin_unlock(needs_lock);
  616. needs_lock = NULL;
  617. } else
  618. *dir = NULL;
  619. nr_pages_to_free++;
  620. list_add(&middir->lru, &pages_to_free);
  621. }
  622. shmem_dir_unmap(dir);
  623. cond_resched();
  624. dir = shmem_dir_map(middir);
  625. diroff = 0;
  626. }
  627. punch_lock = needs_lock;
  628. subdir = dir[diroff];
  629. if (subdir && !offset && upper_limit-idx >= ENTRIES_PER_PAGE) {
  630. if (needs_lock) {
  631. spin_lock(needs_lock);
  632. dir[diroff] = NULL;
  633. spin_unlock(needs_lock);
  634. punch_lock = NULL;
  635. } else
  636. dir[diroff] = NULL;
  637. nr_pages_to_free++;
  638. list_add(&subdir->lru, &pages_to_free);
  639. }
  640. if (subdir && page_private(subdir) /* has swap entries */) {
  641. size = limit - idx;
  642. if (size > ENTRIES_PER_PAGE)
  643. size = ENTRIES_PER_PAGE;
  644. freed = shmem_map_and_free_swp(subdir,
  645. offset, size, &dir, punch_lock);
  646. if (!dir)
  647. dir = shmem_dir_map(middir);
  648. nr_swaps_freed += freed;
  649. if (offset || punch_lock) {
  650. spin_lock(&info->lock);
  651. set_page_private(subdir,
  652. page_private(subdir) - freed);
  653. spin_unlock(&info->lock);
  654. } else
  655. BUG_ON(page_private(subdir) != freed);
  656. }
  657. offset = 0;
  658. }
  659. done1:
  660. shmem_dir_unmap(dir);
  661. done2:
  662. if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) {
  663. /*
  664. * Call truncate_inode_pages again: racing shmem_unuse_inode
  665. * may have swizzled a page in from swap since
  666. * truncate_pagecache or generic_delete_inode did it, before we
  667. * lowered next_index. Also, though shmem_getpage checks
  668. * i_size before adding to cache, no recheck after: so fix the
  669. * narrow window there too.
  670. *
  671. * Recalling truncate_inode_pages_range and unmap_mapping_range
  672. * every time for punch_hole (which never got a chance to clear
  673. * SHMEM_PAGEIN at the start of vmtruncate_range) is expensive,
  674. * yet hardly ever necessary: try to optimize them out later.
  675. */
  676. truncate_inode_pages_range(inode->i_mapping, start, end);
  677. if (punch_hole)
  678. unmap_mapping_range(inode->i_mapping, start,
  679. end - start, 1);
  680. }
  681. spin_lock(&info->lock);
  682. info->flags &= ~SHMEM_TRUNCATE;
  683. info->swapped -= nr_swaps_freed;
  684. if (nr_pages_to_free)
  685. shmem_free_blocks(inode, nr_pages_to_free);
  686. shmem_recalc_inode(inode);
  687. spin_unlock(&info->lock);
  688. /*
  689. * Empty swap vector directory pages to be freed?
  690. */
  691. if (!list_empty(&pages_to_free)) {
  692. pages_to_free.prev->next = NULL;
  693. shmem_free_pages(pages_to_free.next);
  694. }
  695. }
  696. static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
  697. {
  698. struct inode *inode = dentry->d_inode;
  699. loff_t newsize = attr->ia_size;
  700. int error;
  701. error = inode_change_ok(inode, attr);
  702. if (error)
  703. return error;
  704. if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)
  705. && newsize != inode->i_size) {
  706. struct page *page = NULL;
  707. if (newsize < inode->i_size) {
  708. /*
  709. * If truncating down to a partial page, then
  710. * if that page is already allocated, hold it
  711. * in memory until the truncation is over, so
  712. * truncate_partial_page cannot miss it were
  713. * it assigned to swap.
  714. */
  715. if (newsize & (PAGE_CACHE_SIZE-1)) {
  716. (void) shmem_getpage(inode,
  717. newsize >> PAGE_CACHE_SHIFT,
  718. &page, SGP_READ, NULL);
  719. if (page)
  720. unlock_page(page);
  721. }
  722. /*
  723. * Reset SHMEM_PAGEIN flag so that shmem_truncate can
  724. * detect if any pages might have been added to cache
  725. * after truncate_inode_pages. But we needn't bother
  726. * if it's being fully truncated to zero-length: the
  727. * nrpages check is efficient enough in that case.
  728. */
  729. if (newsize) {
  730. struct shmem_inode_info *info = SHMEM_I(inode);
  731. spin_lock(&info->lock);
  732. info->flags &= ~SHMEM_PAGEIN;
  733. spin_unlock(&info->lock);
  734. }
  735. }
  736. /* XXX(truncate): truncate_setsize should be called last */
  737. truncate_setsize(inode, newsize);
  738. if (page)
  739. page_cache_release(page);
  740. shmem_truncate_range(inode, newsize, (loff_t)-1);
  741. }
  742. setattr_copy(inode, attr);
  743. #ifdef CONFIG_TMPFS_POSIX_ACL
  744. if (attr->ia_valid & ATTR_MODE)
  745. error = generic_acl_chmod(inode);
  746. #endif
  747. return error;
  748. }
  749. static void shmem_evict_inode(struct inode *inode)
  750. {
  751. struct shmem_inode_info *info = SHMEM_I(inode);
  752. if (inode->i_mapping->a_ops == &shmem_aops) {
  753. truncate_inode_pages(inode->i_mapping, 0);
  754. shmem_unacct_size(info->flags, inode->i_size);
  755. inode->i_size = 0;
  756. shmem_truncate_range(inode, 0, (loff_t)-1);
  757. if (!list_empty(&info->swaplist)) {
  758. mutex_lock(&shmem_swaplist_mutex);
  759. list_del_init(&info->swaplist);
  760. mutex_unlock(&shmem_swaplist_mutex);
  761. }
  762. }
  763. BUG_ON(inode->i_blocks);
  764. shmem_free_inode(inode->i_sb);
  765. end_writeback(inode);
  766. }
  767. static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
  768. {
  769. swp_entry_t *ptr;
  770. for (ptr = dir; ptr < edir; ptr++) {
  771. if (ptr->val == entry.val)
  772. return ptr - dir;
  773. }
  774. return -1;
  775. }
  776. static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
  777. {
  778. struct address_space *mapping;
  779. unsigned long idx;
  780. unsigned long size;
  781. unsigned long limit;
  782. unsigned long stage;
  783. struct page **dir;
  784. struct page *subdir;
  785. swp_entry_t *ptr;
  786. int offset;
  787. int error;
  788. idx = 0;
  789. ptr = info->i_direct;
  790. spin_lock(&info->lock);
  791. if (!info->swapped) {
  792. list_del_init(&info->swaplist);
  793. goto lost2;
  794. }
  795. limit = info->next_index;
  796. size = limit;
  797. if (size > SHMEM_NR_DIRECT)
  798. size = SHMEM_NR_DIRECT;
  799. offset = shmem_find_swp(entry, ptr, ptr+size);
  800. if (offset >= 0) {
  801. shmem_swp_balance_unmap();
  802. goto found;
  803. }
  804. if (!info->i_indirect)
  805. goto lost2;
  806. dir = shmem_dir_map(info->i_indirect);
  807. stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2;
  808. for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
  809. if (unlikely(idx == stage)) {
  810. shmem_dir_unmap(dir-1);
  811. if (cond_resched_lock(&info->lock)) {
  812. /* check it has not been truncated */
  813. if (limit > info->next_index) {
  814. limit = info->next_index;
  815. if (idx >= limit)
  816. goto lost2;
  817. }
  818. }
  819. dir = shmem_dir_map(info->i_indirect) +
  820. ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
  821. while (!*dir) {
  822. dir++;
  823. idx += ENTRIES_PER_PAGEPAGE;
  824. if (idx >= limit)
  825. goto lost1;
  826. }
  827. stage = idx + ENTRIES_PER_PAGEPAGE;
  828. subdir = *dir;
  829. shmem_dir_unmap(dir);
  830. dir = shmem_dir_map(subdir);
  831. }
  832. subdir = *dir;
  833. if (subdir && page_private(subdir)) {
  834. ptr = shmem_swp_map(subdir);
  835. size = limit - idx;
  836. if (size > ENTRIES_PER_PAGE)
  837. size = ENTRIES_PER_PAGE;
  838. offset = shmem_find_swp(entry, ptr, ptr+size);
  839. shmem_swp_unmap(ptr);
  840. if (offset >= 0) {
  841. shmem_dir_unmap(dir);
  842. ptr = shmem_swp_map(subdir);
  843. goto found;
  844. }
  845. }
  846. }
  847. lost1:
  848. shmem_dir_unmap(dir-1);
  849. lost2:
  850. spin_unlock(&info->lock);
  851. return 0;
  852. found:
  853. idx += offset;
  854. ptr += offset;
  855. /*
  856. * Move _head_ to start search for next from here.
  857. * But be careful: shmem_evict_inode checks list_empty without taking
  858. * mutex, and there's an instant in list_move_tail when info->swaplist
  859. * would appear empty, if it were the only one on shmem_swaplist. We
  860. * could avoid doing it if inode NULL; or use this minor optimization.
  861. */
  862. if (shmem_swaplist.next != &info->swaplist)
  863. list_move_tail(&shmem_swaplist, &info->swaplist);
  864. /*
  865. * We rely on shmem_swaplist_mutex, not only to protect the swaplist,
  866. * but also to hold up shmem_evict_inode(): so inode cannot be freed
  867. * beneath us (pagelock doesn't help until the page is in pagecache).
  868. */
  869. mapping = info->vfs_inode.i_mapping;
  870. error = add_to_page_cache_locked(page, mapping, idx, GFP_NOWAIT);
  871. /* which does mem_cgroup_uncharge_cache_page on error */
  872. if (error == -EEXIST) {
  873. struct page *filepage = find_get_page(mapping, idx);
  874. error = 1;
  875. if (filepage) {
  876. /*
  877. * There might be a more uptodate page coming down
  878. * from a stacked writepage: forget our swappage if so.
  879. */
  880. if (PageUptodate(filepage))
  881. error = 0;
  882. page_cache_release(filepage);
  883. }
  884. }
  885. if (!error) {
  886. delete_from_swap_cache(page);
  887. set_page_dirty(page);
  888. info->flags |= SHMEM_PAGEIN;
  889. shmem_swp_set(info, ptr, 0);
  890. swap_free(entry);
  891. error = 1; /* not an error, but entry was found */
  892. }
  893. shmem_swp_unmap(ptr);
  894. spin_unlock(&info->lock);
  895. return error;
  896. }
  897. /*
  898. * shmem_unuse() search for an eventually swapped out shmem page.
  899. */
  900. int shmem_unuse(swp_entry_t entry, struct page *page)
  901. {
  902. struct list_head *p, *next;
  903. struct shmem_inode_info *info;
  904. int found = 0;
  905. int error;
  906. /*
  907. * Charge page using GFP_KERNEL while we can wait, before taking
  908. * the shmem_swaplist_mutex which might hold up shmem_writepage().
  909. * Charged back to the user (not to caller) when swap account is used.
  910. * add_to_page_cache() will be called with GFP_NOWAIT.
  911. */
  912. error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
  913. if (error)
  914. goto out;
  915. /*
  916. * Try to preload while we can wait, to not make a habit of
  917. * draining atomic reserves; but don't latch on to this cpu,
  918. * it's okay if sometimes we get rescheduled after this.
  919. */
  920. error = radix_tree_preload(GFP_KERNEL);
  921. if (error)
  922. goto uncharge;
  923. radix_tree_preload_end();
  924. mutex_lock(&shmem_swaplist_mutex);
  925. list_for_each_safe(p, next, &shmem_swaplist) {
  926. info = list_entry(p, struct shmem_inode_info, swaplist);
  927. found = shmem_unuse_inode(info, entry, page);
  928. cond_resched();
  929. if (found)
  930. break;
  931. }
  932. mutex_unlock(&shmem_swaplist_mutex);
  933. uncharge:
  934. if (!found)
  935. mem_cgroup_uncharge_cache_page(page);
  936. if (found < 0)
  937. error = found;
  938. out:
  939. unlock_page(page);
  940. page_cache_release(page);
  941. return error;
  942. }
  943. /*
  944. * Move the page from the page cache to the swap cache.
  945. */
  946. static int shmem_writepage(struct page *page, struct writeback_control *wbc)
  947. {
  948. struct shmem_inode_info *info;
  949. swp_entry_t *entry, swap;
  950. struct address_space *mapping;
  951. unsigned long index;
  952. struct inode *inode;
  953. BUG_ON(!PageLocked(page));
  954. mapping = page->mapping;
  955. index = page->index;
  956. inode = mapping->host;
  957. info = SHMEM_I(inode);
  958. if (info->flags & VM_LOCKED)
  959. goto redirty;
  960. if (!total_swap_pages)
  961. goto redirty;
  962. /*
  963. * shmem_backing_dev_info's capabilities prevent regular writeback or
  964. * sync from ever calling shmem_writepage; but a stacking filesystem
  965. * may use the ->writepage of its underlying filesystem, in which case
  966. * tmpfs should write out to swap only in response to memory pressure,
  967. * and not for the writeback threads or sync. However, in those cases,
  968. * we do still want to check if there's a redundant swappage to be
  969. * discarded.
  970. */
  971. if (wbc->for_reclaim)
  972. swap = get_swap_page();
  973. else
  974. swap.val = 0;
  975. /*
  976. * Add inode to shmem_unuse()'s list of swapped-out inodes,
  977. * if it's not already there. Do it now because we cannot take
  978. * mutex while holding spinlock, and must do so before the page
  979. * is moved to swap cache, when its pagelock no longer protects
  980. * the inode from eviction. But don't unlock the mutex until
  981. * we've taken the spinlock, because shmem_unuse_inode() will
  982. * prune a !swapped inode from the swaplist under both locks.
  983. */
  984. if (swap.val) {
  985. mutex_lock(&shmem_swaplist_mutex);
  986. if (list_empty(&info->swaplist))
  987. list_add_tail(&info->swaplist, &shmem_swaplist);
  988. }
  989. spin_lock(&info->lock);
  990. if (swap.val)
  991. mutex_unlock(&shmem_swaplist_mutex);
  992. if (index >= info->next_index) {
  993. BUG_ON(!(info->flags & SHMEM_TRUNCATE));
  994. goto unlock;
  995. }
  996. entry = shmem_swp_entry(info, index, NULL);
  997. if (entry->val) {
  998. /*
  999. * The more uptodate page coming down from a stacked
  1000. * writepage should replace our old swappage.
  1001. */
  1002. free_swap_and_cache(*entry);
  1003. shmem_swp_set(info, entry, 0);
  1004. }
  1005. shmem_recalc_inode(inode);
  1006. if (swap.val && add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
  1007. delete_from_page_cache(page);
  1008. shmem_swp_set(info, entry, swap.val);
  1009. shmem_swp_unmap(entry);
  1010. spin_unlock(&info->lock);
  1011. swap_shmem_alloc(swap);
  1012. BUG_ON(page_mapped(page));
  1013. swap_writepage(page, wbc);
  1014. return 0;
  1015. }
  1016. shmem_swp_unmap(entry);
  1017. unlock:
  1018. spin_unlock(&info->lock);
  1019. /*
  1020. * add_to_swap_cache() doesn't return -EEXIST, so we can safely
  1021. * clear SWAP_HAS_CACHE flag.
  1022. */
  1023. swapcache_free(swap, NULL);
  1024. redirty:
  1025. set_page_dirty(page);
  1026. if (wbc->for_reclaim)
  1027. return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */
  1028. unlock_page(page);
  1029. return 0;
  1030. }
  1031. #ifdef CONFIG_NUMA
  1032. #ifdef CONFIG_TMPFS
  1033. static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
  1034. {
  1035. char buffer[64];
  1036. if (!mpol || mpol->mode == MPOL_DEFAULT)
  1037. return; /* show nothing */
  1038. mpol_to_str(buffer, sizeof(buffer), mpol, 1);
  1039. seq_printf(seq, ",mpol=%s", buffer);
  1040. }
  1041. static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
  1042. {
  1043. struct mempolicy *mpol = NULL;
  1044. if (sbinfo->mpol) {
  1045. spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */
  1046. mpol = sbinfo->mpol;
  1047. mpol_get(mpol);
  1048. spin_unlock(&sbinfo->stat_lock);
  1049. }
  1050. return mpol;
  1051. }
  1052. #endif /* CONFIG_TMPFS */
  1053. static struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
  1054. struct shmem_inode_info *info, unsigned long idx)
  1055. {
  1056. struct mempolicy mpol, *spol;
  1057. struct vm_area_struct pvma;
  1058. struct page *page;
  1059. spol = mpol_cond_copy(&mpol,
  1060. mpol_shared_policy_lookup(&info->policy, idx));
  1061. /* Create a pseudo vma that just contains the policy */
  1062. pvma.vm_start = 0;
  1063. pvma.vm_pgoff = idx;
  1064. pvma.vm_ops = NULL;
  1065. pvma.vm_policy = spol;
  1066. page = swapin_readahead(entry, gfp, &pvma, 0);
  1067. return page;
  1068. }
  1069. static struct page *shmem_alloc_page(gfp_t gfp,
  1070. struct shmem_inode_info *info, unsigned long idx)
  1071. {
  1072. struct vm_area_struct pvma;
  1073. /* Create a pseudo vma that just contains the policy */
  1074. pvma.vm_start = 0;
  1075. pvma.vm_pgoff = idx;
  1076. pvma.vm_ops = NULL;
  1077. pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
  1078. /*
  1079. * alloc_page_vma() will drop the shared policy reference
  1080. */
  1081. return alloc_page_vma(gfp, &pvma, 0);
  1082. }
  1083. #else /* !CONFIG_NUMA */
  1084. #ifdef CONFIG_TMPFS
  1085. static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *p)
  1086. {
  1087. }
  1088. #endif /* CONFIG_TMPFS */
  1089. static inline struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
  1090. struct shmem_inode_info *info, unsigned long idx)
  1091. {
  1092. return swapin_readahead(entry, gfp, NULL, 0);
  1093. }
  1094. static inline struct page *shmem_alloc_page(gfp_t gfp,
  1095. struct shmem_inode_info *info, unsigned long idx)
  1096. {
  1097. return alloc_page(gfp);
  1098. }
  1099. #endif /* CONFIG_NUMA */
  1100. #if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
  1101. static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
  1102. {
  1103. return NULL;
  1104. }
  1105. #endif
  1106. /*
  1107. * shmem_getpage - either get the page from swap or allocate a new one
  1108. *
  1109. * If we allocate a new one we do not mark it dirty. That's up to the
  1110. * vm. If we swap it in we mark it dirty since we also free the swap
  1111. * entry since a page cannot live in both the swap and page cache
  1112. */
  1113. static int shmem_getpage(struct inode *inode, unsigned long idx,
  1114. struct page **pagep, enum sgp_type sgp, int *type)
  1115. {
  1116. struct address_space *mapping = inode->i_mapping;
  1117. struct shmem_inode_info *info = SHMEM_I(inode);
  1118. struct shmem_sb_info *sbinfo;
  1119. struct page *filepage = *pagep;
  1120. struct page *swappage;
  1121. struct page *prealloc_page = NULL;
  1122. swp_entry_t *entry;
  1123. swp_entry_t swap;
  1124. gfp_t gfp;
  1125. int error;
  1126. if (idx >= SHMEM_MAX_INDEX)
  1127. return -EFBIG;
  1128. if (type)
  1129. *type = 0;
  1130. /*
  1131. * Normally, filepage is NULL on entry, and either found
  1132. * uptodate immediately, or allocated and zeroed, or read
  1133. * in under swappage, which is then assigned to filepage.
  1134. * But shmem_readpage (required for splice) passes in a locked
  1135. * filepage, which may be found not uptodate by other callers
  1136. * too, and may need to be copied from the swappage read in.
  1137. */
  1138. repeat:
  1139. if (!filepage)
  1140. filepage = find_lock_page(mapping, idx);
  1141. if (filepage && PageUptodate(filepage))
  1142. goto done;
  1143. gfp = mapping_gfp_mask(mapping);
  1144. if (!filepage) {
  1145. /*
  1146. * Try to preload while we can wait, to not make a habit of
  1147. * draining atomic reserves; but don't latch on to this cpu.
  1148. */
  1149. error = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
  1150. if (error)
  1151. goto failed;
  1152. radix_tree_preload_end();
  1153. if (sgp != SGP_READ && !prealloc_page) {
  1154. /* We don't care if this fails */
  1155. prealloc_page = shmem_alloc_page(gfp, info, idx);
  1156. if (prealloc_page) {
  1157. if (mem_cgroup_cache_charge(prealloc_page,
  1158. current->mm, GFP_KERNEL)) {
  1159. page_cache_release(prealloc_page);
  1160. prealloc_page = NULL;
  1161. }
  1162. }
  1163. }
  1164. }
  1165. error = 0;
  1166. spin_lock(&info->lock);
  1167. shmem_recalc_inode(inode);
  1168. entry = shmem_swp_alloc(info, idx, sgp);
  1169. if (IS_ERR(entry)) {
  1170. spin_unlock(&info->lock);
  1171. error = PTR_ERR(entry);
  1172. goto failed;
  1173. }
  1174. swap = *entry;
  1175. if (swap.val) {
  1176. /* Look it up and read it in.. */
  1177. swappage = lookup_swap_cache(swap);
  1178. if (!swappage) {
  1179. shmem_swp_unmap(entry);
  1180. /* here we actually do the io */
  1181. if (type && !(*type & VM_FAULT_MAJOR)) {
  1182. __count_vm_event(PGMAJFAULT);
  1183. *type |= VM_FAULT_MAJOR;
  1184. }
  1185. spin_unlock(&info->lock);
  1186. swappage = shmem_swapin(swap, gfp, info, idx);
  1187. if (!swappage) {
  1188. spin_lock(&info->lock);
  1189. entry = shmem_swp_alloc(info, idx, sgp);
  1190. if (IS_ERR(entry))
  1191. error = PTR_ERR(entry);
  1192. else {
  1193. if (entry->val == swap.val)
  1194. error = -ENOMEM;
  1195. shmem_swp_unmap(entry);
  1196. }
  1197. spin_unlock(&info->lock);
  1198. if (error)
  1199. goto failed;
  1200. goto repeat;
  1201. }
  1202. wait_on_page_locked(swappage);
  1203. page_cache_release(swappage);
  1204. goto repeat;
  1205. }
  1206. /* We have to do this with page locked to prevent races */
  1207. if (!trylock_page(swappage)) {
  1208. shmem_swp_unmap(entry);
  1209. spin_unlock(&info->lock);
  1210. wait_on_page_locked(swappage);
  1211. page_cache_release(swappage);
  1212. goto repeat;
  1213. }
  1214. if (PageWriteback(swappage)) {
  1215. shmem_swp_unmap(entry);
  1216. spin_unlock(&info->lock);
  1217. wait_on_page_writeback(swappage);
  1218. unlock_page(swappage);
  1219. page_cache_release(swappage);
  1220. goto repeat;
  1221. }
  1222. if (!PageUptodate(swappage)) {
  1223. shmem_swp_unmap(entry);
  1224. spin_unlock(&info->lock);
  1225. unlock_page(swappage);
  1226. page_cache_release(swappage);
  1227. error = -EIO;
  1228. goto failed;
  1229. }
  1230. if (filepage) {
  1231. shmem_swp_set(info, entry, 0);
  1232. shmem_swp_unmap(entry);
  1233. delete_from_swap_cache(swappage);
  1234. spin_unlock(&info->lock);
  1235. copy_highpage(filepage, swappage);
  1236. unlock_page(swappage);
  1237. page_cache_release(swappage);
  1238. flush_dcache_page(filepage);
  1239. SetPageUptodate(filepage);
  1240. set_page_dirty(filepage);
  1241. swap_free(swap);
  1242. } else if (!(error = add_to_page_cache_locked(swappage, mapping,
  1243. idx, GFP_NOWAIT))) {
  1244. info->flags |= SHMEM_PAGEIN;
  1245. shmem_swp_set(info, entry, 0);
  1246. shmem_swp_unmap(entry);
  1247. delete_from_swap_cache(swappage);
  1248. spin_unlock(&info->lock);
  1249. filepage = swappage;
  1250. set_page_dirty(filepage);
  1251. swap_free(swap);
  1252. } else {
  1253. shmem_swp_unmap(entry);
  1254. spin_unlock(&info->lock);
  1255. if (error == -ENOMEM) {
  1256. /*
  1257. * reclaim from proper memory cgroup and
  1258. * call memcg's OOM if needed.
  1259. */
  1260. error = mem_cgroup_shmem_charge_fallback(
  1261. swappage,
  1262. current->mm,
  1263. gfp);
  1264. if (error) {
  1265. unlock_page(swappage);
  1266. page_cache_release(swappage);
  1267. goto failed;
  1268. }
  1269. }
  1270. unlock_page(swappage);
  1271. page_cache_release(swappage);
  1272. goto repeat;
  1273. }
  1274. } else if (sgp == SGP_READ && !filepage) {
  1275. shmem_swp_unmap(entry);
  1276. filepage = find_get_page(mapping, idx);
  1277. if (filepage &&
  1278. (!PageUptodate(filepage) || !trylock_page(filepage))) {
  1279. spin_unlock(&info->lock);
  1280. wait_on_page_locked(filepage);
  1281. page_cache_release(filepage);
  1282. filepage = NULL;
  1283. goto repeat;
  1284. }
  1285. spin_unlock(&info->lock);
  1286. } else {
  1287. shmem_swp_unmap(entry);
  1288. sbinfo = SHMEM_SB(inode->i_sb);
  1289. if (sbinfo->max_blocks) {
  1290. if (percpu_counter_compare(&sbinfo->used_blocks,
  1291. sbinfo->max_blocks) >= 0 ||
  1292. shmem_acct_block(info->flags))
  1293. goto nospace;
  1294. percpu_counter_inc(&sbinfo->used_blocks);
  1295. spin_lock(&inode->i_lock);
  1296. inode->i_blocks += BLOCKS_PER_PAGE;
  1297. spin_unlock(&inode->i_lock);
  1298. } else if (shmem_acct_block(info->flags))
  1299. goto nospace;
  1300. if (!filepage) {
  1301. int ret;
  1302. if (!prealloc_page) {
  1303. spin_unlock(&info->lock);
  1304. filepage = shmem_alloc_page(gfp, info, idx);
  1305. if (!filepage) {
  1306. shmem_unacct_blocks(info->flags, 1);
  1307. shmem_free_blocks(inode, 1);
  1308. error = -ENOMEM;
  1309. goto failed;
  1310. }
  1311. SetPageSwapBacked(filepage);
  1312. /*
  1313. * Precharge page while we can wait, compensate
  1314. * after
  1315. */
  1316. error = mem_cgroup_cache_charge(filepage,
  1317. current->mm, GFP_KERNEL);
  1318. if (error) {
  1319. page_cache_release(filepage);
  1320. shmem_unacct_blocks(info->flags, 1);
  1321. shmem_free_blocks(inode, 1);
  1322. filepage = NULL;
  1323. goto failed;
  1324. }
  1325. spin_lock(&info->lock);
  1326. } else {
  1327. filepage = prealloc_page;
  1328. prealloc_page = NULL;
  1329. SetPageSwapBacked(filepage);
  1330. }
  1331. entry = shmem_swp_alloc(info, idx, sgp);
  1332. if (IS_ERR(entry))
  1333. error = PTR_ERR(entry);
  1334. else {
  1335. swap = *entry;
  1336. shmem_swp_unmap(entry);
  1337. }
  1338. ret = error || swap.val;
  1339. if (ret)
  1340. mem_cgroup_uncharge_cache_page(filepage);
  1341. else
  1342. ret = add_to_page_cache_lru(filepage, mapping,
  1343. idx, GFP_NOWAIT);
  1344. /*
  1345. * At add_to_page_cache_lru() failure, uncharge will
  1346. * be done automatically.
  1347. */
  1348. if (ret) {
  1349. spin_unlock(&info->lock);
  1350. page_cache_release(filepage);
  1351. shmem_unacct_blocks(info->flags, 1);
  1352. shmem_free_blocks(inode, 1);
  1353. filepage = NULL;
  1354. if (error)
  1355. goto failed;
  1356. goto repeat;
  1357. }
  1358. info->flags |= SHMEM_PAGEIN;
  1359. }
  1360. info->alloced++;
  1361. spin_unlock(&info->lock);
  1362. clear_highpage(filepage);
  1363. flush_dcache_page(filepage);
  1364. SetPageUptodate(filepage);
  1365. if (sgp == SGP_DIRTY)
  1366. set_page_dirty(filepage);
  1367. }
  1368. done:
  1369. *pagep = filepage;
  1370. error = 0;
  1371. goto out;
  1372. nospace:
  1373. /*
  1374. * Perhaps the page was brought in from swap between find_lock_page
  1375. * and taking info->lock? We allow for that at add_to_page_cache_lru,
  1376. * but must also avoid reporting a spurious ENOSPC while working on a
  1377. * full tmpfs. (When filepage has been passed in to shmem_getpage, it
  1378. * is already in page cache, which prevents this race from occurring.)
  1379. */
  1380. if (!filepage) {
  1381. struct page *page = find_get_page(mapping, idx);
  1382. if (page) {
  1383. spin_unlock(&info->lock);
  1384. page_cache_release(page);
  1385. goto repeat;
  1386. }
  1387. }
  1388. spin_unlock(&info->lock);
  1389. error = -ENOSPC;
  1390. failed:
  1391. if (*pagep != filepage) {
  1392. unlock_page(filepage);
  1393. page_cache_release(filepage);
  1394. }
  1395. out:
  1396. if (prealloc_page) {
  1397. mem_cgroup_uncharge_cache_page(prealloc_page);
  1398. page_cache_release(prealloc_page);
  1399. }
  1400. return error;
  1401. }
  1402. static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  1403. {
  1404. struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
  1405. int error;
  1406. int ret;
  1407. if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
  1408. return VM_FAULT_SIGBUS;
  1409. error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
  1410. if (error)
  1411. return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
  1412. return ret | VM_FAULT_LOCKED;
  1413. }
  1414. #ifdef CONFIG_NUMA
  1415. static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
  1416. {
  1417. struct inode *i = vma->vm_file->f_path.dentry->d_inode;
  1418. return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new);
  1419. }
  1420. static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
  1421. unsigned long addr)
  1422. {
  1423. struct inode *i = vma->vm_file->f_path.dentry->d_inode;
  1424. unsigned long idx;
  1425. idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
  1426. return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx);
  1427. }
  1428. #endif
  1429. int shmem_lock(struct file *file, int lock, struct user_struct *user)
  1430. {
  1431. struct inode *inode = file->f_path.dentry->d_inode;
  1432. struct shmem_inode_info *info = SHMEM_I(inode);
  1433. int retval = -ENOMEM;
  1434. spin_lock(&info->lock);
  1435. if (lock && !(info->flags & VM_LOCKED)) {
  1436. if (!user_shm_lock(inode->i_size, user))
  1437. goto out_nomem;
  1438. info->flags |= VM_LOCKED;
  1439. mapping_set_unevictable(file->f_mapping);
  1440. }
  1441. if (!lock && (info->flags & VM_LOCKED) && user) {
  1442. user_shm_unlock(inode->i_size, user);
  1443. info->flags &= ~VM_LOCKED;
  1444. mapping_clear_unevictable(file->f_mapping);
  1445. scan_mapping_unevictable_pages(file->f_mapping);
  1446. }
  1447. retval = 0;
  1448. out_nomem:
  1449. spin_unlock(&info->lock);
  1450. return retval;
  1451. }
  1452. static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
  1453. {
  1454. file_accessed(file);
  1455. vma->vm_ops = &shmem_vm_ops;
  1456. vma->vm_flags |= VM_CAN_NONLINEAR;
  1457. return 0;
  1458. }
  1459. static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
  1460. int mode, dev_t dev, unsigned long flags)
  1461. {
  1462. struct inode *inode;
  1463. struct shmem_inode_info *info;
  1464. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  1465. if (shmem_reserve_inode(sb))
  1466. return NULL;
  1467. inode = new_inode(sb);
  1468. if (inode) {
  1469. inode->i_ino = get_next_ino();
  1470. inode_init_owner(inode, dir, mode);
  1471. inode->i_blocks = 0;
  1472. inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
  1473. inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
  1474. inode->i_generation = get_seconds();
  1475. info = SHMEM_I(inode);
  1476. memset(info, 0, (char *)inode - (char *)info);
  1477. spin_lock_init(&info->lock);
  1478. info->flags = flags & VM_NORESERVE;
  1479. INIT_LIST_HEAD(&info->swaplist);
  1480. cache_no_acl(inode);
  1481. switch (mode & S_IFMT) {
  1482. default:
  1483. inode->i_op = &shmem_special_inode_operations;
  1484. init_special_inode(inode, mode, dev);
  1485. break;
  1486. case S_IFREG:
  1487. inode->i_mapping->a_ops = &shmem_aops;
  1488. inode->i_op = &shmem_inode_operations;
  1489. inode->i_fop = &shmem_file_operations;
  1490. mpol_shared_policy_init(&info->policy,
  1491. shmem_get_sbmpol(sbinfo));
  1492. break;
  1493. case S_IFDIR:
  1494. inc_nlink(inode);
  1495. /* Some things misbehave if size == 0 on a directory */
  1496. inode->i_size = 2 * BOGO_DIRENT_SIZE;
  1497. inode->i_op = &shmem_dir_inode_operations;
  1498. inode->i_fop = &simple_dir_operations;
  1499. break;
  1500. case S_IFLNK:
  1501. /*
  1502. * Must not load anything in the rbtree,
  1503. * mpol_free_shared_policy will not be called.
  1504. */
  1505. mpol_shared_policy_init(&info->policy, NULL);
  1506. break;
  1507. }
  1508. } else
  1509. shmem_free_inode(sb);
  1510. return inode;
  1511. }
  1512. #ifdef CONFIG_TMPFS
  1513. static const struct inode_operations shmem_symlink_inode_operations;
  1514. static const struct inode_operations shmem_symlink_inline_operations;
  1515. /*
  1516. * Normally tmpfs avoids the use of shmem_readpage and shmem_write_begin;
  1517. * but providing them allows a tmpfs file to be used for splice, sendfile, and
  1518. * below the loop driver, in the generic fashion that many filesystems support.
  1519. */
  1520. static int shmem_readpage(struct file *file, struct page *page)
  1521. {
  1522. struct inode *inode = page->mapping->host;
  1523. int error = shmem_getpage(inode, page->index, &page, SGP_CACHE, NULL);
  1524. unlock_page(page);
  1525. return error;
  1526. }
  1527. static int
  1528. shmem_write_begin(struct file *file, struct address_space *mapping,
  1529. loff_t pos, unsigned len, unsigned flags,
  1530. struct page **pagep, void **fsdata)
  1531. {
  1532. struct inode *inode = mapping->host;
  1533. pgoff_t index = pos >> PAGE_CACHE_SHIFT;
  1534. *pagep = NULL;
  1535. return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
  1536. }
  1537. static int
  1538. shmem_write_end(struct file *file, struct address_space *mapping,
  1539. loff_t pos, unsigned len, unsigned copied,
  1540. struct page *page, void *fsdata)
  1541. {
  1542. struct inode *inode = mapping->host;
  1543. if (pos + copied > inode->i_size)
  1544. i_size_write(inode, pos + copied);
  1545. set_page_dirty(page);
  1546. unlock_page(page);
  1547. page_cache_release(page);
  1548. return copied;
  1549. }
  1550. static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
  1551. {
  1552. struct inode *inode = filp->f_path.dentry->d_inode;
  1553. struct address_space *mapping = inode->i_mapping;
  1554. unsigned long index, offset;
  1555. enum sgp_type sgp = SGP_READ;
  1556. /*
  1557. * Might this read be for a stacking filesystem? Then when reading
  1558. * holes of a sparse file, we actually need to allocate those pages,
  1559. * and even mark them dirty, so it cannot exceed the max_blocks limit.
  1560. */
  1561. if (segment_eq(get_fs(), KERNEL_DS))
  1562. sgp = SGP_DIRTY;
  1563. index = *ppos >> PAGE_CACHE_SHIFT;
  1564. offset = *ppos & ~PAGE_CACHE_MASK;
  1565. for (;;) {
  1566. struct page *page = NULL;
  1567. unsigned long end_index, nr, ret;
  1568. loff_t i_size = i_size_read(inode);
  1569. end_index = i_size >> PAGE_CACHE_SHIFT;
  1570. if (index > end_index)
  1571. break;
  1572. if (index == end_index) {
  1573. nr = i_size & ~PAGE_CACHE_MASK;
  1574. if (nr <= offset)
  1575. break;
  1576. }
  1577. desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
  1578. if (desc->error) {
  1579. if (desc->error == -EINVAL)
  1580. desc->error = 0;
  1581. break;
  1582. }
  1583. if (page)
  1584. unlock_page(page);
  1585. /*
  1586. * We must evaluate after, since reads (unlike writes)
  1587. * are called without i_mutex protection against truncate
  1588. */
  1589. nr = PAGE_CACHE_SIZE;
  1590. i_size = i_size_read(inode);
  1591. end_index = i_size >> PAGE_CACHE_SHIFT;
  1592. if (index == end_index) {
  1593. nr = i_size & ~PAGE_CACHE_MASK;
  1594. if (nr <= offset) {
  1595. if (page)
  1596. page_cache_release(page);
  1597. break;
  1598. }
  1599. }
  1600. nr -= offset;
  1601. if (page) {
  1602. /*
  1603. * If users can be writing to this page using arbitrary
  1604. * virtual addresses, take care about potential aliasing
  1605. * before reading the page on the kernel side.
  1606. */
  1607. if (mapping_writably_mapped(mapping))
  1608. flush_dcache_page(page);
  1609. /*
  1610. * Mark the page accessed if we read the beginning.
  1611. */
  1612. if (!offset)
  1613. mark_page_accessed(page);
  1614. } else {
  1615. page = ZERO_PAGE(0);
  1616. page_cache_get(page);
  1617. }
  1618. /*
  1619. * Ok, we have the page, and it's up-to-date, so
  1620. * now we can copy it to user space...
  1621. *
  1622. * The actor routine returns how many bytes were actually used..
  1623. * NOTE! This may not be the same as how much of a user buffer
  1624. * we filled up (we may be padding etc), so we can only update
  1625. * "pos" here (the actor routine has to update the user buffer
  1626. * pointers and the remaining count).
  1627. */
  1628. ret = actor(desc, page, offset, nr);
  1629. offset += ret;
  1630. index += offset >> PAGE_CACHE_SHIFT;
  1631. offset &= ~PAGE_CACHE_MASK;
  1632. page_cache_release(page);
  1633. if (ret != nr || !desc->count)
  1634. break;
  1635. cond_resched();
  1636. }
  1637. *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
  1638. file_accessed(filp);
  1639. }
  1640. static ssize_t shmem_file_aio_read(struct kiocb *iocb,
  1641. const struct iovec *iov, unsigned long nr_segs, loff_t pos)
  1642. {
  1643. struct file *filp = iocb->ki_filp;
  1644. ssize_t retval;
  1645. unsigned long seg;
  1646. size_t count;
  1647. loff_t *ppos = &iocb->ki_pos;
  1648. retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
  1649. if (retval)
  1650. return retval;
  1651. for (seg = 0; seg < nr_segs; seg++) {
  1652. read_descriptor_t desc;
  1653. desc.written = 0;
  1654. desc.arg.buf = iov[seg].iov_base;
  1655. desc.count = iov[seg].iov_len;
  1656. if (desc.count == 0)
  1657. continue;
  1658. desc.error = 0;
  1659. do_shmem_file_read(filp, ppos, &desc, file_read_actor);
  1660. retval += desc.written;
  1661. if (desc.error) {
  1662. retval = retval ?: desc.error;
  1663. break;
  1664. }
  1665. if (desc.count > 0)
  1666. break;
  1667. }
  1668. return retval;
  1669. }
  1670. static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
  1671. {
  1672. struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
  1673. buf->f_type = TMPFS_MAGIC;
  1674. buf->f_bsize = PAGE_CACHE_SIZE;
  1675. buf->f_namelen = NAME_MAX;
  1676. if (sbinfo->max_blocks) {
  1677. buf->f_blocks = sbinfo->max_blocks;
  1678. buf->f_bavail = buf->f_bfree =
  1679. sbinfo->max_blocks - percpu_counter_sum(&sbinfo->used_blocks);
  1680. }
  1681. if (sbinfo->max_inodes) {
  1682. buf->f_files = sbinfo->max_inodes;
  1683. buf->f_ffree = sbinfo->free_inodes;
  1684. }
  1685. /* else leave those fields 0 like simple_statfs */
  1686. return 0;
  1687. }
  1688. /*
  1689. * File creation. Allocate an inode, and we're done..
  1690. */
  1691. static int
  1692. shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
  1693. {
  1694. struct inode *inode;
  1695. int error = -ENOSPC;
  1696. inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
  1697. if (inode) {
  1698. error = security_inode_init_security(inode, dir,
  1699. &dentry->d_name, NULL,
  1700. NULL, NULL);
  1701. if (error) {
  1702. if (error != -EOPNOTSUPP) {
  1703. iput(inode);
  1704. return error;
  1705. }
  1706. }
  1707. #ifdef CONFIG_TMPFS_POSIX_ACL
  1708. error = generic_acl_init(inode, dir);
  1709. if (error) {
  1710. iput(inode);
  1711. return error;
  1712. }
  1713. #else
  1714. error = 0;
  1715. #endif
  1716. dir->i_size += BOGO_DIRENT_SIZE;
  1717. dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  1718. d_instantiate(dentry, inode);
  1719. dget(dentry); /* Extra count - pin the dentry in core */
  1720. }
  1721. return error;
  1722. }
  1723. static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
  1724. {
  1725. int error;
  1726. if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
  1727. return error;
  1728. inc_nlink(dir);
  1729. return 0;
  1730. }
  1731. static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
  1732. struct nameidata *nd)
  1733. {
  1734. return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
  1735. }
  1736. /*
  1737. * Link a file..
  1738. */
  1739. static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
  1740. {
  1741. struct inode *inode = old_dentry->d_inode;
  1742. int ret;
  1743. /*
  1744. * No ordinary (disk based) filesystem counts links as inodes;
  1745. * but each new link needs a new dentry, pinning lowmem, and
  1746. * tmpfs dentries cannot be pruned until they are unlinked.
  1747. */
  1748. ret = shmem_reserve_inode(inode->i_sb);
  1749. if (ret)
  1750. goto out;
  1751. dir->i_size += BOGO_DIRENT_SIZE;
  1752. inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  1753. inc_nlink(inode);
  1754. ihold(inode); /* New dentry reference */
  1755. dget(dentry); /* Extra pinning count for the created dentry */
  1756. d_instantiate(dentry, inode);
  1757. out:
  1758. return ret;
  1759. }
  1760. static int shmem_unlink(struct inode *dir, struct dentry *dentry)
  1761. {
  1762. struct inode *inode = dentry->d_inode;
  1763. if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
  1764. shmem_free_inode(inode->i_sb);
  1765. dir->i_size -= BOGO_DIRENT_SIZE;
  1766. inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  1767. drop_nlink(inode);
  1768. dput(dentry); /* Undo the count from "create" - this does all the work */
  1769. return 0;
  1770. }
  1771. static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
  1772. {
  1773. if (!simple_empty(dentry))
  1774. return -ENOTEMPTY;
  1775. drop_nlink(dentry->d_inode);
  1776. drop_nlink(dir);
  1777. return shmem_unlink(dir, dentry);
  1778. }
  1779. /*
  1780. * The VFS layer already does all the dentry stuff for rename,
  1781. * we just have to decrement the usage count for the target if
  1782. * it exists so that the VFS layer correctly free's it when it
  1783. * gets overwritten.
  1784. */
  1785. static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
  1786. {
  1787. struct inode *inode = old_dentry->d_inode;
  1788. int they_are_dirs = S_ISDIR(inode->i_mode);
  1789. if (!simple_empty(new_dentry))
  1790. return -ENOTEMPTY;
  1791. if (new_dentry->d_inode) {
  1792. (void) shmem_unlink(new_dir, new_dentry);
  1793. if (they_are_dirs)
  1794. drop_nlink(old_dir);
  1795. } else if (they_are_dirs) {
  1796. drop_nlink(old_dir);
  1797. inc_nlink(new_dir);
  1798. }
  1799. old_dir->i_size -= BOGO_DIRENT_SIZE;
  1800. new_dir->i_size += BOGO_DIRENT_SIZE;
  1801. old_dir->i_ctime = old_dir->i_mtime =
  1802. new_dir->i_ctime = new_dir->i_mtime =
  1803. inode->i_ctime = CURRENT_TIME;
  1804. return 0;
  1805. }
  1806. static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
  1807. {
  1808. int error;
  1809. int len;
  1810. struct inode *inode;
  1811. struct page *page = NULL;
  1812. char *kaddr;
  1813. struct shmem_inode_info *info;
  1814. len = strlen(symname) + 1;
  1815. if (len > PAGE_CACHE_SIZE)
  1816. return -ENAMETOOLONG;
  1817. inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
  1818. if (!inode)
  1819. return -ENOSPC;
  1820. error = security_inode_init_security(inode, dir, &dentry->d_name, NULL,
  1821. NULL, NULL);
  1822. if (error) {
  1823. if (error != -EOPNOTSUPP) {
  1824. iput(inode);
  1825. return error;
  1826. }
  1827. error = 0;
  1828. }
  1829. info = SHMEM_I(inode);
  1830. inode->i_size = len-1;
  1831. if (len <= (char *)inode - (char *)info) {
  1832. /* do it inline */
  1833. memcpy(info, symname, len);
  1834. inode->i_op = &shmem_symlink_inline_operations;
  1835. } else {
  1836. error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
  1837. if (error) {
  1838. iput(inode);
  1839. return error;
  1840. }
  1841. inode->i_mapping->a_ops = &shmem_aops;
  1842. inode->i_op = &shmem_symlink_inode_operations;
  1843. kaddr = kmap_atomic(page, KM_USER0);
  1844. memcpy(kaddr, symname, len);
  1845. kunmap_atomic(kaddr, KM_USER0);
  1846. set_page_dirty(page);
  1847. unlock_page(page);
  1848. page_cache_release(page);
  1849. }
  1850. dir->i_size += BOGO_DIRENT_SIZE;
  1851. dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  1852. d_instantiate(dentry, inode);
  1853. dget(dentry);
  1854. return 0;
  1855. }
  1856. static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
  1857. {
  1858. nd_set_link(nd, (char *)SHMEM_I(dentry->d_inode));
  1859. return NULL;
  1860. }
  1861. static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
  1862. {
  1863. struct page *page = NULL;
  1864. int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
  1865. nd_set_link(nd, res ? ERR_PTR(res) : kmap(page));
  1866. if (page)
  1867. unlock_page(page);
  1868. return page;
  1869. }
  1870. static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
  1871. {
  1872. if (!IS_ERR(nd_get_link(nd))) {
  1873. struct page *page = cookie;
  1874. kunmap(page);
  1875. mark_page_accessed(page);
  1876. page_cache_release(page);
  1877. }
  1878. }
  1879. static const struct inode_operations shmem_symlink_inline_operations = {
  1880. .readlink = generic_readlink,
  1881. .follow_link = shmem_follow_link_inline,
  1882. };
  1883. static const struct inode_operations shmem_symlink_inode_operations = {
  1884. .readlink = generic_readlink,
  1885. .follow_link = shmem_follow_link,
  1886. .put_link = shmem_put_link,
  1887. };
  1888. #ifdef CONFIG_TMPFS_POSIX_ACL
  1889. /*
  1890. * Superblocks without xattr inode operations will get security.* xattr
  1891. * support from the VFS "for free". As soon as we have any other xattrs
  1892. * like ACLs, we also need to implement the security.* handlers at
  1893. * filesystem level, though.
  1894. */
  1895. static size_t shmem_xattr_security_list(struct dentry *dentry, char *list,
  1896. size_t list_len, const char *name,
  1897. size_t name_len, int handler_flags)
  1898. {
  1899. return security_inode_listsecurity(dentry->d_inode, list, list_len);
  1900. }
  1901. static int shmem_xattr_security_get(struct dentry *dentry, const char *name,
  1902. void *buffer, size_t size, int handler_flags)
  1903. {
  1904. if (strcmp(name, "") == 0)
  1905. return -EINVAL;
  1906. return xattr_getsecurity(dentry->d_inode, name, buffer, size);
  1907. }
  1908. static int shmem_xattr_security_set(struct dentry *dentry, const char *name,
  1909. const void *value, size_t size, int flags, int handler_flags)
  1910. {
  1911. if (strcmp(name, "") == 0)
  1912. return -EINVAL;
  1913. return security_inode_setsecurity(dentry->d_inode, name, value,
  1914. size, flags);
  1915. }
  1916. static const struct xattr_handler shmem_xattr_security_handler = {
  1917. .prefix = XATTR_SECURITY_PREFIX,
  1918. .list = shmem_xattr_security_list,
  1919. .get = shmem_xattr_security_get,
  1920. .set = shmem_xattr_security_set,
  1921. };
  1922. static const struct xattr_handler *shmem_xattr_handlers[] = {
  1923. &generic_acl_access_handler,
  1924. &generic_acl_default_handler,
  1925. &shmem_xattr_security_handler,
  1926. NULL
  1927. };
  1928. #endif
  1929. static struct dentry *shmem_get_parent(struct dentry *child)
  1930. {
  1931. return ERR_PTR(-ESTALE);
  1932. }
  1933. static int shmem_match(struct inode *ino, void *vfh)
  1934. {
  1935. __u32 *fh = vfh;
  1936. __u64 inum = fh[2];
  1937. inum = (inum << 32) | fh[1];
  1938. return ino->i_ino == inum && fh[0] == ino->i_generation;
  1939. }
  1940. static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
  1941. struct fid *fid, int fh_len, int fh_type)
  1942. {
  1943. struct inode *inode;
  1944. struct dentry *dentry = NULL;
  1945. u64 inum = fid->raw[2];
  1946. inum = (inum << 32) | fid->raw[1];
  1947. if (fh_len < 3)
  1948. return NULL;
  1949. inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
  1950. shmem_match, fid->raw);
  1951. if (inode) {
  1952. dentry = d_find_alias(inode);
  1953. iput(inode);
  1954. }
  1955. return dentry;
  1956. }
  1957. static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
  1958. int connectable)
  1959. {
  1960. struct inode *inode = dentry->d_inode;
  1961. if (*len < 3) {
  1962. *len = 3;
  1963. return 255;
  1964. }
  1965. if (inode_unhashed(inode)) {
  1966. /* Unfortunately insert_inode_hash is not idempotent,
  1967. * so as we hash inodes here rather than at creation
  1968. * time, we need a lock to ensure we only try
  1969. * to do it once
  1970. */
  1971. static DEFINE_SPINLOCK(lock);
  1972. spin_lock(&lock);
  1973. if (inode_unhashed(inode))
  1974. __insert_inode_hash(inode,
  1975. inode->i_ino + inode->i_generation);
  1976. spin_unlock(&lock);
  1977. }
  1978. fh[0] = inode->i_generation;
  1979. fh[1] = inode->i_ino;
  1980. fh[2] = ((__u64)inode->i_ino) >> 32;
  1981. *len = 3;
  1982. return 1;
  1983. }
  1984. static const struct export_operations shmem_export_ops = {
  1985. .get_parent = shmem_get_parent,
  1986. .encode_fh = shmem_encode_fh,
  1987. .fh_to_dentry = shmem_fh_to_dentry,
  1988. };
  1989. static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
  1990. bool remount)
  1991. {
  1992. char *this_char, *value, *rest;
  1993. while (options != NULL) {
  1994. this_char = options;
  1995. for (;;) {
  1996. /*
  1997. * NUL-terminate this option: unfortunately,
  1998. * mount options form a comma-separated list,
  1999. * but mpol's nodelist may also contain commas.
  2000. */
  2001. options = strchr(options, ',');
  2002. if (options == NULL)
  2003. break;
  2004. options++;
  2005. if (!isdigit(*options)) {
  2006. options[-1] = '\0';
  2007. break;
  2008. }
  2009. }
  2010. if (!*this_char)
  2011. continue;
  2012. if ((value = strchr(this_char,'=')) != NULL) {
  2013. *value++ = 0;
  2014. } else {
  2015. printk(KERN_ERR
  2016. "tmpfs: No value for mount option '%s'\n",
  2017. this_char);
  2018. return 1;
  2019. }
  2020. if (!strcmp(this_char,"size")) {
  2021. unsigned long long size;
  2022. size = memparse(value,&rest);
  2023. if (*rest == '%') {
  2024. size <<= PAGE_SHIFT;
  2025. size *= totalram_pages;
  2026. do_div(size, 100);
  2027. rest++;
  2028. }
  2029. if (*rest)
  2030. goto bad_val;
  2031. sbinfo->max_blocks =
  2032. DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
  2033. } else if (!strcmp(this_char,"nr_blocks")) {
  2034. sbinfo->max_blocks = memparse(value, &rest);
  2035. if (*rest)
  2036. goto bad_val;
  2037. } else if (!strcmp(this_char,"nr_inodes")) {
  2038. sbinfo->max_inodes = memparse(value, &rest);
  2039. if (*rest)
  2040. goto bad_val;
  2041. } else if (!strcmp(this_char,"mode")) {
  2042. if (remount)
  2043. continue;
  2044. sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
  2045. if (*rest)
  2046. goto bad_val;
  2047. } else if (!strcmp(this_char,"uid")) {
  2048. if (remount)
  2049. continue;
  2050. sbinfo->uid = simple_strtoul(value, &rest, 0);
  2051. if (*rest)
  2052. goto bad_val;
  2053. } else if (!strcmp(this_char,"gid")) {
  2054. if (remount)
  2055. continue;
  2056. sbinfo->gid = simple_strtoul(value, &rest, 0);
  2057. if (*rest)
  2058. goto bad_val;
  2059. } else if (!strcmp(this_char,"mpol")) {
  2060. if (mpol_parse_str(value, &sbinfo->mpol, 1))
  2061. goto bad_val;
  2062. } else {
  2063. printk(KERN_ERR "tmpfs: Bad mount option %s\n",
  2064. this_char);
  2065. return 1;
  2066. }
  2067. }
  2068. return 0;
  2069. bad_val:
  2070. printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
  2071. value, this_char);
  2072. return 1;
  2073. }
  2074. static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
  2075. {
  2076. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  2077. struct shmem_sb_info config = *sbinfo;
  2078. unsigned long inodes;
  2079. int error = -EINVAL;
  2080. if (shmem_parse_options(data, &config, true))
  2081. return error;
  2082. spin_lock(&sbinfo->stat_lock);
  2083. inodes = sbinfo->max_inodes - sbinfo->free_inodes;
  2084. if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
  2085. goto out;
  2086. if (config.max_inodes < inodes)
  2087. goto out;
  2088. /*
  2089. * Those tests also disallow limited->unlimited while any are in
  2090. * use, so i_blocks will always be zero when max_blocks is zero;
  2091. * but we must separately disallow unlimited->limited, because
  2092. * in that case we have no record of how much is already in use.
  2093. */
  2094. if (config.max_blocks && !sbinfo->max_blocks)
  2095. goto out;
  2096. if (config.max_inodes && !sbinfo->max_inodes)
  2097. goto out;
  2098. error = 0;
  2099. sbinfo->max_blocks = config.max_blocks;
  2100. sbinfo->max_inodes = config.max_inodes;
  2101. sbinfo->free_inodes = config.max_inodes - inodes;
  2102. mpol_put(sbinfo->mpol);
  2103. sbinfo->mpol = config.mpol; /* transfers initial ref */
  2104. out:
  2105. spin_unlock(&sbinfo->stat_lock);
  2106. return error;
  2107. }
  2108. static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs)
  2109. {
  2110. struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb);
  2111. if (sbinfo->max_blocks != shmem_default_max_blocks())
  2112. seq_printf(seq, ",size=%luk",
  2113. sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
  2114. if (sbinfo->max_inodes != shmem_default_max_inodes())
  2115. seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
  2116. if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
  2117. seq_printf(seq, ",mode=%03o", sbinfo->mode);
  2118. if (sbinfo->uid != 0)
  2119. seq_printf(seq, ",uid=%u", sbinfo->uid);
  2120. if (sbinfo->gid != 0)
  2121. seq_printf(seq, ",gid=%u", sbinfo->gid);
  2122. shmem_show_mpol(seq, sbinfo->mpol);
  2123. return 0;
  2124. }
  2125. #endif /* CONFIG_TMPFS */
  2126. static void shmem_put_super(struct super_block *sb)
  2127. {
  2128. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  2129. percpu_counter_destroy(&sbinfo->used_blocks);
  2130. kfree(sbinfo);
  2131. sb->s_fs_info = NULL;
  2132. }
  2133. int shmem_fill_super(struct super_block *sb, void *data, int silent)
  2134. {
  2135. struct inode *inode;
  2136. struct dentry *root;
  2137. struct shmem_sb_info *sbinfo;
  2138. int err = -ENOMEM;
  2139. /* Round up to L1_CACHE_BYTES to resist false sharing */
  2140. sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
  2141. L1_CACHE_BYTES), GFP_KERNEL);
  2142. if (!sbinfo)
  2143. return -ENOMEM;
  2144. sbinfo->mode = S_IRWXUGO | S_ISVTX;
  2145. sbinfo->uid = current_fsuid();
  2146. sbinfo->gid = current_fsgid();
  2147. sb->s_fs_info = sbinfo;
  2148. #ifdef CONFIG_TMPFS
  2149. /*
  2150. * Per default we only allow half of the physical ram per
  2151. * tmpfs instance, limiting inodes to one per page of lowmem;
  2152. * but the internal instance is left unlimited.
  2153. */
  2154. if (!(sb->s_flags & MS_NOUSER)) {
  2155. sbinfo->max_blocks = shmem_default_max_blocks();
  2156. sbinfo->max_inodes = shmem_default_max_inodes();
  2157. if (shmem_parse_options(data, sbinfo, false)) {
  2158. err = -EINVAL;
  2159. goto failed;
  2160. }
  2161. }
  2162. sb->s_export_op = &shmem_export_ops;
  2163. #else
  2164. sb->s_flags |= MS_NOUSER;
  2165. #endif
  2166. spin_lock_init(&sbinfo->stat_lock);
  2167. if (percpu_counter_init(&sbinfo->used_blocks, 0))
  2168. goto failed;
  2169. sbinfo->free_inodes = sbinfo->max_inodes;
  2170. sb->s_maxbytes = SHMEM_MAX_BYTES;
  2171. sb->s_blocksize = PAGE_CACHE_SIZE;
  2172. sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
  2173. sb->s_magic = TMPFS_MAGIC;
  2174. sb->s_op = &shmem_ops;
  2175. sb->s_time_gran = 1;
  2176. #ifdef CONFIG_TMPFS_POSIX_ACL
  2177. sb->s_xattr = shmem_xattr_handlers;
  2178. sb->s_flags |= MS_POSIXACL;
  2179. #endif
  2180. inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
  2181. if (!inode)
  2182. goto failed;
  2183. inode->i_uid = sbinfo->uid;
  2184. inode->i_gid = sbinfo->gid;
  2185. root = d_alloc_root(inode);
  2186. if (!root)
  2187. goto failed_iput;
  2188. sb->s_root = root;
  2189. return 0;
  2190. failed_iput:
  2191. iput(inode);
  2192. failed:
  2193. shmem_put_super(sb);
  2194. return err;
  2195. }
  2196. static struct kmem_cache *shmem_inode_cachep;
  2197. static struct inode *shmem_alloc_inode(struct super_block *sb)
  2198. {
  2199. struct shmem_inode_info *p;
  2200. p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
  2201. if (!p)
  2202. return NULL;
  2203. return &p->vfs_inode;
  2204. }
  2205. static void shmem_i_callback(struct rcu_head *head)
  2206. {
  2207. struct inode *inode = container_of(head, struct inode, i_rcu);
  2208. INIT_LIST_HEAD(&inode->i_dentry);
  2209. kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
  2210. }
  2211. static void shmem_destroy_inode(struct inode *inode)
  2212. {
  2213. if ((inode->i_mode & S_IFMT) == S_IFREG) {
  2214. /* only struct inode is valid if it's an inline symlink */
  2215. mpol_free_shared_policy(&SHMEM_I(inode)->policy);
  2216. }
  2217. call_rcu(&inode->i_rcu, shmem_i_callback);
  2218. }
  2219. static void init_once(void *foo)
  2220. {
  2221. struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
  2222. inode_init_once(&p->vfs_inode);
  2223. }
  2224. static int init_inodecache(void)
  2225. {
  2226. shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
  2227. sizeof(struct shmem_inode_info),
  2228. 0, SLAB_PANIC, init_once);
  2229. return 0;
  2230. }
  2231. static void destroy_inodecache(void)
  2232. {
  2233. kmem_cache_destroy(shmem_inode_cachep);
  2234. }
  2235. static const struct address_space_operations shmem_aops = {
  2236. .writepage = shmem_writepage,
  2237. .set_page_dirty = __set_page_dirty_no_writeback,
  2238. #ifdef CONFIG_TMPFS
  2239. .readpage = shmem_readpage,
  2240. .write_begin = shmem_write_begin,
  2241. .write_end = shmem_write_end,
  2242. #endif
  2243. .migratepage = migrate_page,
  2244. .error_remove_page = generic_error_remove_page,
  2245. };
  2246. static const struct file_operations shmem_file_operations = {
  2247. .mmap = shmem_mmap,
  2248. #ifdef CONFIG_TMPFS
  2249. .llseek = generic_file_llseek,
  2250. .read = do_sync_read,
  2251. .write = do_sync_write,
  2252. .aio_read = shmem_file_aio_read,
  2253. .aio_write = generic_file_aio_write,
  2254. .fsync = noop_fsync,
  2255. .splice_read = generic_file_splice_read,
  2256. .splice_write = generic_file_splice_write,
  2257. #endif
  2258. };
  2259. static const struct inode_operations shmem_inode_operations = {
  2260. .setattr = shmem_notify_change,
  2261. .truncate_range = shmem_truncate_range,
  2262. #ifdef CONFIG_TMPFS_POSIX_ACL
  2263. .setxattr = generic_setxattr,
  2264. .getxattr = generic_getxattr,
  2265. .listxattr = generic_listxattr,
  2266. .removexattr = generic_removexattr,
  2267. .check_acl = generic_check_acl,
  2268. #endif
  2269. };
  2270. static const struct inode_operations shmem_dir_inode_operations = {
  2271. #ifdef CONFIG_TMPFS
  2272. .create = shmem_create,
  2273. .lookup = simple_lookup,
  2274. .link = shmem_link,
  2275. .unlink = shmem_unlink,
  2276. .symlink = shmem_symlink,
  2277. .mkdir = shmem_mkdir,
  2278. .rmdir = shmem_rmdir,
  2279. .mknod = shmem_mknod,
  2280. .rename = shmem_rename,
  2281. #endif
  2282. #ifdef CONFIG_TMPFS_POSIX_ACL
  2283. .setattr = shmem_notify_change,
  2284. .setxattr = generic_setxattr,
  2285. .getxattr = generic_getxattr,
  2286. .listxattr = generic_listxattr,
  2287. .removexattr = generic_removexattr,
  2288. .check_acl = generic_check_acl,
  2289. #endif
  2290. };
  2291. static const struct inode_operations shmem_special_inode_operations = {
  2292. #ifdef CONFIG_TMPFS_POSIX_ACL
  2293. .setattr = shmem_notify_change,
  2294. .setxattr = generic_setxattr,
  2295. .getxattr = generic_getxattr,
  2296. .listxattr = generic_listxattr,
  2297. .removexattr = generic_removexattr,
  2298. .check_acl = generic_check_acl,
  2299. #endif
  2300. };
  2301. static const struct super_operations shmem_ops = {
  2302. .alloc_inode = shmem_alloc_inode,
  2303. .destroy_inode = shmem_destroy_inode,
  2304. #ifdef CONFIG_TMPFS
  2305. .statfs = shmem_statfs,
  2306. .remount_fs = shmem_remount_fs,
  2307. .show_options = shmem_show_options,
  2308. #endif
  2309. .evict_inode = shmem_evict_inode,
  2310. .drop_inode = generic_delete_inode,
  2311. .put_super = shmem_put_super,
  2312. };
  2313. static const struct vm_operations_struct shmem_vm_ops = {
  2314. .fault = shmem_fault,
  2315. #ifdef CONFIG_NUMA
  2316. .set_policy = shmem_set_policy,
  2317. .get_policy = shmem_get_policy,
  2318. #endif
  2319. };
  2320. static struct dentry *shmem_mount(struct file_system_type *fs_type,
  2321. int flags, const char *dev_name, void *data)
  2322. {
  2323. return mount_nodev(fs_type, flags, data, shmem_fill_super);
  2324. }
  2325. static struct file_system_type tmpfs_fs_type = {
  2326. .owner = THIS_MODULE,
  2327. .name = "tmpfs",
  2328. .mount = shmem_mount,
  2329. .kill_sb = kill_litter_super,
  2330. };
  2331. int __init init_tmpfs(void)
  2332. {
  2333. int error;
  2334. error = bdi_init(&shmem_backing_dev_info);
  2335. if (error)
  2336. goto out4;
  2337. error = init_inodecache();
  2338. if (error)
  2339. goto out3;
  2340. error = register_filesystem(&tmpfs_fs_type);
  2341. if (error) {
  2342. printk(KERN_ERR "Could not register tmpfs\n");
  2343. goto out2;
  2344. }
  2345. shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER,
  2346. tmpfs_fs_type.name, NULL);
  2347. if (IS_ERR(shm_mnt)) {
  2348. error = PTR_ERR(shm_mnt);
  2349. printk(KERN_ERR "Could not kern_mount tmpfs\n");
  2350. goto out1;
  2351. }
  2352. return 0;
  2353. out1:
  2354. unregister_filesystem(&tmpfs_fs_type);
  2355. out2:
  2356. destroy_inodecache();
  2357. out3:
  2358. bdi_destroy(&shmem_backing_dev_info);
  2359. out4:
  2360. shm_mnt = ERR_PTR(error);
  2361. return error;
  2362. }
  2363. #ifdef CONFIG_CGROUP_MEM_RES_CTLR
  2364. /**
  2365. * mem_cgroup_get_shmem_target - find a page or entry assigned to the shmem file
  2366. * @inode: the inode to be searched
  2367. * @pgoff: the offset to be searched
  2368. * @pagep: the pointer for the found page to be stored
  2369. * @ent: the pointer for the found swap entry to be stored
  2370. *
  2371. * If a page is found, refcount of it is incremented. Callers should handle
  2372. * these refcount.
  2373. */
  2374. void mem_cgroup_get_shmem_target(struct inode *inode, pgoff_t pgoff,
  2375. struct page **pagep, swp_entry_t *ent)
  2376. {
  2377. swp_entry_t entry = { .val = 0 }, *ptr;
  2378. struct page *page = NULL;
  2379. struct shmem_inode_info *info = SHMEM_I(inode);
  2380. if ((pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
  2381. goto out;
  2382. spin_lock(&info->lock);
  2383. ptr = shmem_swp_entry(info, pgoff, NULL);
  2384. #ifdef CONFIG_SWAP
  2385. if (ptr && ptr->val) {
  2386. entry.val = ptr->val;
  2387. page = find_get_page(&swapper_space, entry.val);
  2388. } else
  2389. #endif
  2390. page = find_get_page(inode->i_mapping, pgoff);
  2391. if (ptr)
  2392. shmem_swp_unmap(ptr);
  2393. spin_unlock(&info->lock);
  2394. out:
  2395. *pagep = page;
  2396. *ent = entry;
  2397. }
  2398. #endif
  2399. #else /* !CONFIG_SHMEM */
  2400. /*
  2401. * tiny-shmem: simple shmemfs and tmpfs using ramfs code
  2402. *
  2403. * This is intended for small system where the benefits of the full
  2404. * shmem code (swap-backed and resource-limited) are outweighed by
  2405. * their complexity. On systems without swap this code should be
  2406. * effectively equivalent, but much lighter weight.
  2407. */
  2408. #include <linux/ramfs.h>
  2409. static struct file_system_type tmpfs_fs_type = {
  2410. .name = "tmpfs",
  2411. .mount = ramfs_mount,
  2412. .kill_sb = kill_litter_super,
  2413. };
  2414. int __init init_tmpfs(void)
  2415. {
  2416. BUG_ON(register_filesystem(&tmpfs_fs_type) != 0);
  2417. shm_mnt = kern_mount(&tmpfs_fs_type);
  2418. BUG_ON(IS_ERR(shm_mnt));
  2419. return 0;
  2420. }
  2421. int shmem_unuse(swp_entry_t entry, struct page *page)
  2422. {
  2423. return 0;
  2424. }
  2425. int shmem_lock(struct file *file, int lock, struct user_struct *user)
  2426. {
  2427. return 0;
  2428. }
  2429. #ifdef CONFIG_CGROUP_MEM_RES_CTLR
  2430. /**
  2431. * mem_cgroup_get_shmem_target - find a page or entry assigned to the shmem file
  2432. * @inode: the inode to be searched
  2433. * @pgoff: the offset to be searched
  2434. * @pagep: the pointer for the found page to be stored
  2435. * @ent: the pointer for the found swap entry to be stored
  2436. *
  2437. * If a page is found, refcount of it is incremented. Callers should handle
  2438. * these refcount.
  2439. */
  2440. void mem_cgroup_get_shmem_target(struct inode *inode, pgoff_t pgoff,
  2441. struct page **pagep, swp_entry_t *ent)
  2442. {
  2443. struct page *page = NULL;
  2444. if ((pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
  2445. goto out;
  2446. page = find_get_page(inode->i_mapping, pgoff);
  2447. out:
  2448. *pagep = page;
  2449. *ent = (swp_entry_t){ .val = 0 };
  2450. }
  2451. #endif
  2452. #define shmem_vm_ops generic_file_vm_ops
  2453. #define shmem_file_operations ramfs_file_operations
  2454. #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev)
  2455. #define shmem_acct_size(flags, size) 0
  2456. #define shmem_unacct_size(flags, size) do {} while (0)
  2457. #define SHMEM_MAX_BYTES MAX_LFS_FILESIZE
  2458. #endif /* CONFIG_SHMEM */
  2459. /* common code */
  2460. /**
  2461. * shmem_file_setup - get an unlinked file living in tmpfs
  2462. * @name: name for dentry (to be seen in /proc/<pid>/maps
  2463. * @size: size to be set for the file
  2464. * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
  2465. */
  2466. struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
  2467. {
  2468. int error;
  2469. struct file *file;
  2470. struct inode *inode;
  2471. struct path path;
  2472. struct dentry *root;
  2473. struct qstr this;
  2474. if (IS_ERR(shm_mnt))
  2475. return (void *)shm_mnt;
  2476. if (size < 0 || size > SHMEM_MAX_BYTES)
  2477. return ERR_PTR(-EINVAL);
  2478. if (shmem_acct_size(flags, size))
  2479. return ERR_PTR(-ENOMEM);
  2480. error = -ENOMEM;
  2481. this.name = name;
  2482. this.len = strlen(name);
  2483. this.hash = 0; /* will go */
  2484. root = shm_mnt->mnt_root;
  2485. path.dentry = d_alloc(root, &this);
  2486. if (!path.dentry)
  2487. goto put_memory;
  2488. path.mnt = mntget(shm_mnt);
  2489. error = -ENOSPC;
  2490. inode = shmem_get_inode(root->d_sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
  2491. if (!inode)
  2492. goto put_dentry;
  2493. d_instantiate(path.dentry, inode);
  2494. inode->i_size = size;
  2495. inode->i_nlink = 0; /* It is unlinked */
  2496. #ifndef CONFIG_MMU
  2497. error = ramfs_nommu_expand_for_mapping(inode, size);
  2498. if (error)
  2499. goto put_dentry;
  2500. #endif
  2501. error = -ENFILE;
  2502. file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
  2503. &shmem_file_operations);
  2504. if (!file)
  2505. goto put_dentry;
  2506. return file;
  2507. put_dentry:
  2508. path_put(&path);
  2509. put_memory:
  2510. shmem_unacct_size(flags, size);
  2511. return ERR_PTR(error);
  2512. }
  2513. EXPORT_SYMBOL_GPL(shmem_file_setup);
  2514. /**
  2515. * shmem_zero_setup - setup a shared anonymous mapping
  2516. * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
  2517. */
  2518. int shmem_zero_setup(struct vm_area_struct *vma)
  2519. {
  2520. struct file *file;
  2521. loff_t size = vma->vm_end - vma->vm_start;
  2522. file = shmem_file_setup("dev/zero", size, vma->vm_flags);
  2523. if (IS_ERR(file))
  2524. return PTR_ERR(file);
  2525. if (vma->vm_file)
  2526. fput(vma->vm_file);
  2527. vma->vm_file = file;
  2528. vma->vm_ops = &shmem_vm_ops;
  2529. vma->vm_flags |= VM_CAN_NONLINEAR;
  2530. return 0;
  2531. }