file.c 62 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436
  1. /*
  2. * Copyright (C) 2007 Oracle. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public
  6. * License v2 as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. * General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public
  14. * License along with this program; if not, write to the
  15. * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16. * Boston, MA 021110-1307, USA.
  17. */
  18. #include <linux/fs.h>
  19. #include <linux/pagemap.h>
  20. #include <linux/highmem.h>
  21. #include <linux/time.h>
  22. #include <linux/init.h>
  23. #include <linux/string.h>
  24. #include <linux/backing-dev.h>
  25. #include <linux/mpage.h>
  26. #include <linux/falloc.h>
  27. #include <linux/swap.h>
  28. #include <linux/writeback.h>
  29. #include <linux/statfs.h>
  30. #include <linux/compat.h>
  31. #include <linux/slab.h>
  32. #include "ctree.h"
  33. #include "disk-io.h"
  34. #include "transaction.h"
  35. #include "btrfs_inode.h"
  36. #include "ioctl.h"
  37. #include "print-tree.h"
  38. #include "tree-log.h"
  39. #include "locking.h"
  40. #include "compat.h"
  41. #include "volumes.h"
  42. static struct kmem_cache *btrfs_inode_defrag_cachep;
  43. /*
  44. * when auto defrag is enabled we
  45. * queue up these defrag structs to remember which
  46. * inodes need defragging passes
  47. */
  48. struct inode_defrag {
  49. struct rb_node rb_node;
  50. /* objectid */
  51. u64 ino;
  52. /*
  53. * transid where the defrag was added, we search for
  54. * extents newer than this
  55. */
  56. u64 transid;
  57. /* root objectid */
  58. u64 root;
  59. /* last offset we were able to defrag */
  60. u64 last_offset;
  61. /* if we've wrapped around back to zero once already */
  62. int cycled;
  63. };
  64. static int __compare_inode_defrag(struct inode_defrag *defrag1,
  65. struct inode_defrag *defrag2)
  66. {
  67. if (defrag1->root > defrag2->root)
  68. return 1;
  69. else if (defrag1->root < defrag2->root)
  70. return -1;
  71. else if (defrag1->ino > defrag2->ino)
  72. return 1;
  73. else if (defrag1->ino < defrag2->ino)
  74. return -1;
  75. else
  76. return 0;
  77. }
  78. /* pop a record for an inode into the defrag tree. The lock
  79. * must be held already
  80. *
  81. * If you're inserting a record for an older transid than an
  82. * existing record, the transid already in the tree is lowered
  83. *
  84. * If an existing record is found the defrag item you
  85. * pass in is freed
  86. */
  87. static int __btrfs_add_inode_defrag(struct inode *inode,
  88. struct inode_defrag *defrag)
  89. {
  90. struct btrfs_root *root = BTRFS_I(inode)->root;
  91. struct inode_defrag *entry;
  92. struct rb_node **p;
  93. struct rb_node *parent = NULL;
  94. int ret;
  95. p = &root->fs_info->defrag_inodes.rb_node;
  96. while (*p) {
  97. parent = *p;
  98. entry = rb_entry(parent, struct inode_defrag, rb_node);
  99. ret = __compare_inode_defrag(defrag, entry);
  100. if (ret < 0)
  101. p = &parent->rb_left;
  102. else if (ret > 0)
  103. p = &parent->rb_right;
  104. else {
  105. /* if we're reinserting an entry for
  106. * an old defrag run, make sure to
  107. * lower the transid of our existing record
  108. */
  109. if (defrag->transid < entry->transid)
  110. entry->transid = defrag->transid;
  111. if (defrag->last_offset > entry->last_offset)
  112. entry->last_offset = defrag->last_offset;
  113. return -EEXIST;
  114. }
  115. }
  116. set_bit(BTRFS_INODE_IN_DEFRAG, &BTRFS_I(inode)->runtime_flags);
  117. rb_link_node(&defrag->rb_node, parent, p);
  118. rb_insert_color(&defrag->rb_node, &root->fs_info->defrag_inodes);
  119. return 0;
  120. }
  121. static inline int __need_auto_defrag(struct btrfs_root *root)
  122. {
  123. if (!btrfs_test_opt(root, AUTO_DEFRAG))
  124. return 0;
  125. if (btrfs_fs_closing(root->fs_info))
  126. return 0;
  127. return 1;
  128. }
  129. /*
  130. * insert a defrag record for this inode if auto defrag is
  131. * enabled
  132. */
  133. int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
  134. struct inode *inode)
  135. {
  136. struct btrfs_root *root = BTRFS_I(inode)->root;
  137. struct inode_defrag *defrag;
  138. u64 transid;
  139. int ret;
  140. if (!__need_auto_defrag(root))
  141. return 0;
  142. if (test_bit(BTRFS_INODE_IN_DEFRAG, &BTRFS_I(inode)->runtime_flags))
  143. return 0;
  144. if (trans)
  145. transid = trans->transid;
  146. else
  147. transid = BTRFS_I(inode)->root->last_trans;
  148. defrag = kmem_cache_zalloc(btrfs_inode_defrag_cachep, GFP_NOFS);
  149. if (!defrag)
  150. return -ENOMEM;
  151. defrag->ino = btrfs_ino(inode);
  152. defrag->transid = transid;
  153. defrag->root = root->root_key.objectid;
  154. spin_lock(&root->fs_info->defrag_inodes_lock);
  155. if (!test_bit(BTRFS_INODE_IN_DEFRAG, &BTRFS_I(inode)->runtime_flags)) {
  156. /*
  157. * If we set IN_DEFRAG flag and evict the inode from memory,
  158. * and then re-read this inode, this new inode doesn't have
  159. * IN_DEFRAG flag. At the case, we may find the existed defrag.
  160. */
  161. ret = __btrfs_add_inode_defrag(inode, defrag);
  162. if (ret)
  163. kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
  164. } else {
  165. kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
  166. }
  167. spin_unlock(&root->fs_info->defrag_inodes_lock);
  168. return 0;
  169. }
  170. /*
  171. * Requeue the defrag object. If there is a defrag object that points to
  172. * the same inode in the tree, we will merge them together (by
  173. * __btrfs_add_inode_defrag()) and free the one that we want to requeue.
  174. */
  175. void btrfs_requeue_inode_defrag(struct inode *inode,
  176. struct inode_defrag *defrag)
  177. {
  178. struct btrfs_root *root = BTRFS_I(inode)->root;
  179. int ret;
  180. if (!__need_auto_defrag(root))
  181. goto out;
  182. /*
  183. * Here we don't check the IN_DEFRAG flag, because we need merge
  184. * them together.
  185. */
  186. spin_lock(&root->fs_info->defrag_inodes_lock);
  187. ret = __btrfs_add_inode_defrag(inode, defrag);
  188. spin_unlock(&root->fs_info->defrag_inodes_lock);
  189. if (ret)
  190. goto out;
  191. return;
  192. out:
  193. kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
  194. }
  195. /*
  196. * pick the defragable inode that we want, if it doesn't exist, we will get
  197. * the next one.
  198. */
  199. static struct inode_defrag *
  200. btrfs_pick_defrag_inode(struct btrfs_fs_info *fs_info, u64 root, u64 ino)
  201. {
  202. struct inode_defrag *entry = NULL;
  203. struct inode_defrag tmp;
  204. struct rb_node *p;
  205. struct rb_node *parent = NULL;
  206. int ret;
  207. tmp.ino = ino;
  208. tmp.root = root;
  209. spin_lock(&fs_info->defrag_inodes_lock);
  210. p = fs_info->defrag_inodes.rb_node;
  211. while (p) {
  212. parent = p;
  213. entry = rb_entry(parent, struct inode_defrag, rb_node);
  214. ret = __compare_inode_defrag(&tmp, entry);
  215. if (ret < 0)
  216. p = parent->rb_left;
  217. else if (ret > 0)
  218. p = parent->rb_right;
  219. else
  220. goto out;
  221. }
  222. if (parent && __compare_inode_defrag(&tmp, entry) > 0) {
  223. parent = rb_next(parent);
  224. if (parent)
  225. entry = rb_entry(parent, struct inode_defrag, rb_node);
  226. else
  227. entry = NULL;
  228. }
  229. out:
  230. if (entry)
  231. rb_erase(parent, &fs_info->defrag_inodes);
  232. spin_unlock(&fs_info->defrag_inodes_lock);
  233. return entry;
  234. }
  235. void btrfs_cleanup_defrag_inodes(struct btrfs_fs_info *fs_info)
  236. {
  237. struct inode_defrag *defrag;
  238. struct rb_node *node;
  239. spin_lock(&fs_info->defrag_inodes_lock);
  240. node = rb_first(&fs_info->defrag_inodes);
  241. while (node) {
  242. rb_erase(node, &fs_info->defrag_inodes);
  243. defrag = rb_entry(node, struct inode_defrag, rb_node);
  244. kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
  245. if (need_resched()) {
  246. spin_unlock(&fs_info->defrag_inodes_lock);
  247. cond_resched();
  248. spin_lock(&fs_info->defrag_inodes_lock);
  249. }
  250. node = rb_first(&fs_info->defrag_inodes);
  251. }
  252. spin_unlock(&fs_info->defrag_inodes_lock);
  253. }
  254. #define BTRFS_DEFRAG_BATCH 1024
  255. static int __btrfs_run_defrag_inode(struct btrfs_fs_info *fs_info,
  256. struct inode_defrag *defrag)
  257. {
  258. struct btrfs_root *inode_root;
  259. struct inode *inode;
  260. struct btrfs_key key;
  261. struct btrfs_ioctl_defrag_range_args range;
  262. int num_defrag;
  263. int index;
  264. int ret;
  265. /* get the inode */
  266. key.objectid = defrag->root;
  267. btrfs_set_key_type(&key, BTRFS_ROOT_ITEM_KEY);
  268. key.offset = (u64)-1;
  269. index = srcu_read_lock(&fs_info->subvol_srcu);
  270. inode_root = btrfs_read_fs_root_no_name(fs_info, &key);
  271. if (IS_ERR(inode_root)) {
  272. ret = PTR_ERR(inode_root);
  273. goto cleanup;
  274. }
  275. if (btrfs_root_refs(&inode_root->root_item) == 0) {
  276. ret = -ENOENT;
  277. goto cleanup;
  278. }
  279. key.objectid = defrag->ino;
  280. btrfs_set_key_type(&key, BTRFS_INODE_ITEM_KEY);
  281. key.offset = 0;
  282. inode = btrfs_iget(fs_info->sb, &key, inode_root, NULL);
  283. if (IS_ERR(inode)) {
  284. ret = PTR_ERR(inode);
  285. goto cleanup;
  286. }
  287. srcu_read_unlock(&fs_info->subvol_srcu, index);
  288. /* do a chunk of defrag */
  289. clear_bit(BTRFS_INODE_IN_DEFRAG, &BTRFS_I(inode)->runtime_flags);
  290. memset(&range, 0, sizeof(range));
  291. range.len = (u64)-1;
  292. range.start = defrag->last_offset;
  293. sb_start_write(fs_info->sb);
  294. num_defrag = btrfs_defrag_file(inode, NULL, &range, defrag->transid,
  295. BTRFS_DEFRAG_BATCH);
  296. sb_end_write(fs_info->sb);
  297. /*
  298. * if we filled the whole defrag batch, there
  299. * must be more work to do. Queue this defrag
  300. * again
  301. */
  302. if (num_defrag == BTRFS_DEFRAG_BATCH) {
  303. defrag->last_offset = range.start;
  304. btrfs_requeue_inode_defrag(inode, defrag);
  305. } else if (defrag->last_offset && !defrag->cycled) {
  306. /*
  307. * we didn't fill our defrag batch, but
  308. * we didn't start at zero. Make sure we loop
  309. * around to the start of the file.
  310. */
  311. defrag->last_offset = 0;
  312. defrag->cycled = 1;
  313. btrfs_requeue_inode_defrag(inode, defrag);
  314. } else {
  315. kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
  316. }
  317. iput(inode);
  318. return 0;
  319. cleanup:
  320. srcu_read_unlock(&fs_info->subvol_srcu, index);
  321. kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
  322. return ret;
  323. }
  324. /*
  325. * run through the list of inodes in the FS that need
  326. * defragging
  327. */
  328. int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info)
  329. {
  330. struct inode_defrag *defrag;
  331. u64 first_ino = 0;
  332. u64 root_objectid = 0;
  333. atomic_inc(&fs_info->defrag_running);
  334. while(1) {
  335. if (!__need_auto_defrag(fs_info->tree_root))
  336. break;
  337. /* find an inode to defrag */
  338. defrag = btrfs_pick_defrag_inode(fs_info, root_objectid,
  339. first_ino);
  340. if (!defrag) {
  341. if (root_objectid || first_ino) {
  342. root_objectid = 0;
  343. first_ino = 0;
  344. continue;
  345. } else {
  346. break;
  347. }
  348. }
  349. first_ino = defrag->ino + 1;
  350. root_objectid = defrag->root;
  351. __btrfs_run_defrag_inode(fs_info, defrag);
  352. }
  353. atomic_dec(&fs_info->defrag_running);
  354. /*
  355. * during unmount, we use the transaction_wait queue to
  356. * wait for the defragger to stop
  357. */
  358. wake_up(&fs_info->transaction_wait);
  359. return 0;
  360. }
  361. /* simple helper to fault in pages and copy. This should go away
  362. * and be replaced with calls into generic code.
  363. */
  364. static noinline int btrfs_copy_from_user(loff_t pos, int num_pages,
  365. size_t write_bytes,
  366. struct page **prepared_pages,
  367. struct iov_iter *i)
  368. {
  369. size_t copied = 0;
  370. size_t total_copied = 0;
  371. int pg = 0;
  372. int offset = pos & (PAGE_CACHE_SIZE - 1);
  373. while (write_bytes > 0) {
  374. size_t count = min_t(size_t,
  375. PAGE_CACHE_SIZE - offset, write_bytes);
  376. struct page *page = prepared_pages[pg];
  377. /*
  378. * Copy data from userspace to the current page
  379. *
  380. * Disable pagefault to avoid recursive lock since
  381. * the pages are already locked
  382. */
  383. pagefault_disable();
  384. copied = iov_iter_copy_from_user_atomic(page, i, offset, count);
  385. pagefault_enable();
  386. /* Flush processor's dcache for this page */
  387. flush_dcache_page(page);
  388. /*
  389. * if we get a partial write, we can end up with
  390. * partially up to date pages. These add
  391. * a lot of complexity, so make sure they don't
  392. * happen by forcing this copy to be retried.
  393. *
  394. * The rest of the btrfs_file_write code will fall
  395. * back to page at a time copies after we return 0.
  396. */
  397. if (!PageUptodate(page) && copied < count)
  398. copied = 0;
  399. iov_iter_advance(i, copied);
  400. write_bytes -= copied;
  401. total_copied += copied;
  402. /* Return to btrfs_file_aio_write to fault page */
  403. if (unlikely(copied == 0))
  404. break;
  405. if (unlikely(copied < PAGE_CACHE_SIZE - offset)) {
  406. offset += copied;
  407. } else {
  408. pg++;
  409. offset = 0;
  410. }
  411. }
  412. return total_copied;
  413. }
  414. /*
  415. * unlocks pages after btrfs_file_write is done with them
  416. */
  417. void btrfs_drop_pages(struct page **pages, size_t num_pages)
  418. {
  419. size_t i;
  420. for (i = 0; i < num_pages; i++) {
  421. /* page checked is some magic around finding pages that
  422. * have been modified without going through btrfs_set_page_dirty
  423. * clear it here
  424. */
  425. ClearPageChecked(pages[i]);
  426. unlock_page(pages[i]);
  427. mark_page_accessed(pages[i]);
  428. page_cache_release(pages[i]);
  429. }
  430. }
  431. /*
  432. * after copy_from_user, pages need to be dirtied and we need to make
  433. * sure holes are created between the current EOF and the start of
  434. * any next extents (if required).
  435. *
  436. * this also makes the decision about creating an inline extent vs
  437. * doing real data extents, marking pages dirty and delalloc as required.
  438. */
  439. int btrfs_dirty_pages(struct btrfs_root *root, struct inode *inode,
  440. struct page **pages, size_t num_pages,
  441. loff_t pos, size_t write_bytes,
  442. struct extent_state **cached)
  443. {
  444. int err = 0;
  445. int i;
  446. u64 num_bytes;
  447. u64 start_pos;
  448. u64 end_of_last_block;
  449. u64 end_pos = pos + write_bytes;
  450. loff_t isize = i_size_read(inode);
  451. start_pos = pos & ~((u64)root->sectorsize - 1);
  452. num_bytes = (write_bytes + pos - start_pos +
  453. root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
  454. end_of_last_block = start_pos + num_bytes - 1;
  455. err = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block,
  456. cached);
  457. if (err)
  458. return err;
  459. for (i = 0; i < num_pages; i++) {
  460. struct page *p = pages[i];
  461. SetPageUptodate(p);
  462. ClearPageChecked(p);
  463. set_page_dirty(p);
  464. }
  465. /*
  466. * we've only changed i_size in ram, and we haven't updated
  467. * the disk i_size. There is no need to log the inode
  468. * at this time.
  469. */
  470. if (end_pos > isize)
  471. i_size_write(inode, end_pos);
  472. return 0;
  473. }
  474. /*
  475. * this drops all the extents in the cache that intersect the range
  476. * [start, end]. Existing extents are split as required.
  477. */
  478. void btrfs_drop_extent_cache(struct inode *inode, u64 start, u64 end,
  479. int skip_pinned)
  480. {
  481. struct extent_map *em;
  482. struct extent_map *split = NULL;
  483. struct extent_map *split2 = NULL;
  484. struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
  485. u64 len = end - start + 1;
  486. u64 gen;
  487. int ret;
  488. int testend = 1;
  489. unsigned long flags;
  490. int compressed = 0;
  491. WARN_ON(end < start);
  492. if (end == (u64)-1) {
  493. len = (u64)-1;
  494. testend = 0;
  495. }
  496. while (1) {
  497. int no_splits = 0;
  498. if (!split)
  499. split = alloc_extent_map();
  500. if (!split2)
  501. split2 = alloc_extent_map();
  502. if (!split || !split2)
  503. no_splits = 1;
  504. write_lock(&em_tree->lock);
  505. em = lookup_extent_mapping(em_tree, start, len);
  506. if (!em) {
  507. write_unlock(&em_tree->lock);
  508. break;
  509. }
  510. flags = em->flags;
  511. gen = em->generation;
  512. if (skip_pinned && test_bit(EXTENT_FLAG_PINNED, &em->flags)) {
  513. if (testend && em->start + em->len >= start + len) {
  514. free_extent_map(em);
  515. write_unlock(&em_tree->lock);
  516. break;
  517. }
  518. start = em->start + em->len;
  519. if (testend)
  520. len = start + len - (em->start + em->len);
  521. free_extent_map(em);
  522. write_unlock(&em_tree->lock);
  523. continue;
  524. }
  525. compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
  526. clear_bit(EXTENT_FLAG_PINNED, &em->flags);
  527. remove_extent_mapping(em_tree, em);
  528. if (no_splits)
  529. goto next;
  530. if (em->block_start < EXTENT_MAP_LAST_BYTE &&
  531. em->start < start) {
  532. split->start = em->start;
  533. split->len = start - em->start;
  534. split->orig_start = em->orig_start;
  535. split->block_start = em->block_start;
  536. if (compressed)
  537. split->block_len = em->block_len;
  538. else
  539. split->block_len = split->len;
  540. split->orig_block_len = max(split->block_len,
  541. em->orig_block_len);
  542. split->generation = gen;
  543. split->bdev = em->bdev;
  544. split->flags = flags;
  545. split->compress_type = em->compress_type;
  546. ret = add_extent_mapping(em_tree, split);
  547. BUG_ON(ret); /* Logic error */
  548. list_move(&split->list, &em_tree->modified_extents);
  549. free_extent_map(split);
  550. split = split2;
  551. split2 = NULL;
  552. }
  553. if (em->block_start < EXTENT_MAP_LAST_BYTE &&
  554. testend && em->start + em->len > start + len) {
  555. u64 diff = start + len - em->start;
  556. split->start = start + len;
  557. split->len = em->start + em->len - (start + len);
  558. split->bdev = em->bdev;
  559. split->flags = flags;
  560. split->compress_type = em->compress_type;
  561. split->generation = gen;
  562. split->orig_block_len = max(em->block_len,
  563. em->orig_block_len);
  564. if (compressed) {
  565. split->block_len = em->block_len;
  566. split->block_start = em->block_start;
  567. split->orig_start = em->orig_start;
  568. } else {
  569. split->block_len = split->len;
  570. split->block_start = em->block_start + diff;
  571. split->orig_start = em->orig_start;
  572. }
  573. ret = add_extent_mapping(em_tree, split);
  574. BUG_ON(ret); /* Logic error */
  575. list_move(&split->list, &em_tree->modified_extents);
  576. free_extent_map(split);
  577. split = NULL;
  578. }
  579. next:
  580. write_unlock(&em_tree->lock);
  581. /* once for us */
  582. free_extent_map(em);
  583. /* once for the tree*/
  584. free_extent_map(em);
  585. }
  586. if (split)
  587. free_extent_map(split);
  588. if (split2)
  589. free_extent_map(split2);
  590. }
  591. /*
  592. * this is very complex, but the basic idea is to drop all extents
  593. * in the range start - end. hint_block is filled in with a block number
  594. * that would be a good hint to the block allocator for this file.
  595. *
  596. * If an extent intersects the range but is not entirely inside the range
  597. * it is either truncated or split. Anything entirely inside the range
  598. * is deleted from the tree.
  599. */
  600. int __btrfs_drop_extents(struct btrfs_trans_handle *trans,
  601. struct btrfs_root *root, struct inode *inode,
  602. struct btrfs_path *path, u64 start, u64 end,
  603. u64 *drop_end, int drop_cache)
  604. {
  605. struct extent_buffer *leaf;
  606. struct btrfs_file_extent_item *fi;
  607. struct btrfs_key key;
  608. struct btrfs_key new_key;
  609. u64 ino = btrfs_ino(inode);
  610. u64 search_start = start;
  611. u64 disk_bytenr = 0;
  612. u64 num_bytes = 0;
  613. u64 extent_offset = 0;
  614. u64 extent_end = 0;
  615. int del_nr = 0;
  616. int del_slot = 0;
  617. int extent_type;
  618. int recow;
  619. int ret;
  620. int modify_tree = -1;
  621. int update_refs = (root->ref_cows || root == root->fs_info->tree_root);
  622. int found = 0;
  623. if (drop_cache)
  624. btrfs_drop_extent_cache(inode, start, end - 1, 0);
  625. if (start >= BTRFS_I(inode)->disk_i_size)
  626. modify_tree = 0;
  627. while (1) {
  628. recow = 0;
  629. ret = btrfs_lookup_file_extent(trans, root, path, ino,
  630. search_start, modify_tree);
  631. if (ret < 0)
  632. break;
  633. if (ret > 0 && path->slots[0] > 0 && search_start == start) {
  634. leaf = path->nodes[0];
  635. btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
  636. if (key.objectid == ino &&
  637. key.type == BTRFS_EXTENT_DATA_KEY)
  638. path->slots[0]--;
  639. }
  640. ret = 0;
  641. next_slot:
  642. leaf = path->nodes[0];
  643. if (path->slots[0] >= btrfs_header_nritems(leaf)) {
  644. BUG_ON(del_nr > 0);
  645. ret = btrfs_next_leaf(root, path);
  646. if (ret < 0)
  647. break;
  648. if (ret > 0) {
  649. ret = 0;
  650. break;
  651. }
  652. leaf = path->nodes[0];
  653. recow = 1;
  654. }
  655. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  656. if (key.objectid > ino ||
  657. key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= end)
  658. break;
  659. fi = btrfs_item_ptr(leaf, path->slots[0],
  660. struct btrfs_file_extent_item);
  661. extent_type = btrfs_file_extent_type(leaf, fi);
  662. if (extent_type == BTRFS_FILE_EXTENT_REG ||
  663. extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
  664. disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  665. num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
  666. extent_offset = btrfs_file_extent_offset(leaf, fi);
  667. extent_end = key.offset +
  668. btrfs_file_extent_num_bytes(leaf, fi);
  669. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  670. extent_end = key.offset +
  671. btrfs_file_extent_inline_len(leaf, fi);
  672. } else {
  673. WARN_ON(1);
  674. extent_end = search_start;
  675. }
  676. if (extent_end <= search_start) {
  677. path->slots[0]++;
  678. goto next_slot;
  679. }
  680. found = 1;
  681. search_start = max(key.offset, start);
  682. if (recow || !modify_tree) {
  683. modify_tree = -1;
  684. btrfs_release_path(path);
  685. continue;
  686. }
  687. /*
  688. * | - range to drop - |
  689. * | -------- extent -------- |
  690. */
  691. if (start > key.offset && end < extent_end) {
  692. BUG_ON(del_nr > 0);
  693. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  694. memcpy(&new_key, &key, sizeof(new_key));
  695. new_key.offset = start;
  696. ret = btrfs_duplicate_item(trans, root, path,
  697. &new_key);
  698. if (ret == -EAGAIN) {
  699. btrfs_release_path(path);
  700. continue;
  701. }
  702. if (ret < 0)
  703. break;
  704. leaf = path->nodes[0];
  705. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  706. struct btrfs_file_extent_item);
  707. btrfs_set_file_extent_num_bytes(leaf, fi,
  708. start - key.offset);
  709. fi = btrfs_item_ptr(leaf, path->slots[0],
  710. struct btrfs_file_extent_item);
  711. extent_offset += start - key.offset;
  712. btrfs_set_file_extent_offset(leaf, fi, extent_offset);
  713. btrfs_set_file_extent_num_bytes(leaf, fi,
  714. extent_end - start);
  715. btrfs_mark_buffer_dirty(leaf);
  716. if (update_refs && disk_bytenr > 0) {
  717. ret = btrfs_inc_extent_ref(trans, root,
  718. disk_bytenr, num_bytes, 0,
  719. root->root_key.objectid,
  720. new_key.objectid,
  721. start - extent_offset, 0);
  722. BUG_ON(ret); /* -ENOMEM */
  723. }
  724. key.offset = start;
  725. }
  726. /*
  727. * | ---- range to drop ----- |
  728. * | -------- extent -------- |
  729. */
  730. if (start <= key.offset && end < extent_end) {
  731. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  732. memcpy(&new_key, &key, sizeof(new_key));
  733. new_key.offset = end;
  734. btrfs_set_item_key_safe(trans, root, path, &new_key);
  735. extent_offset += end - key.offset;
  736. btrfs_set_file_extent_offset(leaf, fi, extent_offset);
  737. btrfs_set_file_extent_num_bytes(leaf, fi,
  738. extent_end - end);
  739. btrfs_mark_buffer_dirty(leaf);
  740. if (update_refs && disk_bytenr > 0)
  741. inode_sub_bytes(inode, end - key.offset);
  742. break;
  743. }
  744. search_start = extent_end;
  745. /*
  746. * | ---- range to drop ----- |
  747. * | -------- extent -------- |
  748. */
  749. if (start > key.offset && end >= extent_end) {
  750. BUG_ON(del_nr > 0);
  751. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  752. btrfs_set_file_extent_num_bytes(leaf, fi,
  753. start - key.offset);
  754. btrfs_mark_buffer_dirty(leaf);
  755. if (update_refs && disk_bytenr > 0)
  756. inode_sub_bytes(inode, extent_end - start);
  757. if (end == extent_end)
  758. break;
  759. path->slots[0]++;
  760. goto next_slot;
  761. }
  762. /*
  763. * | ---- range to drop ----- |
  764. * | ------ extent ------ |
  765. */
  766. if (start <= key.offset && end >= extent_end) {
  767. if (del_nr == 0) {
  768. del_slot = path->slots[0];
  769. del_nr = 1;
  770. } else {
  771. BUG_ON(del_slot + del_nr != path->slots[0]);
  772. del_nr++;
  773. }
  774. if (update_refs &&
  775. extent_type == BTRFS_FILE_EXTENT_INLINE) {
  776. inode_sub_bytes(inode,
  777. extent_end - key.offset);
  778. extent_end = ALIGN(extent_end,
  779. root->sectorsize);
  780. } else if (update_refs && disk_bytenr > 0) {
  781. ret = btrfs_free_extent(trans, root,
  782. disk_bytenr, num_bytes, 0,
  783. root->root_key.objectid,
  784. key.objectid, key.offset -
  785. extent_offset, 0);
  786. BUG_ON(ret); /* -ENOMEM */
  787. inode_sub_bytes(inode,
  788. extent_end - key.offset);
  789. }
  790. if (end == extent_end)
  791. break;
  792. if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {
  793. path->slots[0]++;
  794. goto next_slot;
  795. }
  796. ret = btrfs_del_items(trans, root, path, del_slot,
  797. del_nr);
  798. if (ret) {
  799. btrfs_abort_transaction(trans, root, ret);
  800. break;
  801. }
  802. del_nr = 0;
  803. del_slot = 0;
  804. btrfs_release_path(path);
  805. continue;
  806. }
  807. BUG_ON(1);
  808. }
  809. if (!ret && del_nr > 0) {
  810. ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
  811. if (ret)
  812. btrfs_abort_transaction(trans, root, ret);
  813. }
  814. if (drop_end)
  815. *drop_end = found ? min(end, extent_end) : end;
  816. btrfs_release_path(path);
  817. return ret;
  818. }
  819. int btrfs_drop_extents(struct btrfs_trans_handle *trans,
  820. struct btrfs_root *root, struct inode *inode, u64 start,
  821. u64 end, int drop_cache)
  822. {
  823. struct btrfs_path *path;
  824. int ret;
  825. path = btrfs_alloc_path();
  826. if (!path)
  827. return -ENOMEM;
  828. ret = __btrfs_drop_extents(trans, root, inode, path, start, end, NULL,
  829. drop_cache);
  830. btrfs_free_path(path);
  831. return ret;
  832. }
  833. static int extent_mergeable(struct extent_buffer *leaf, int slot,
  834. u64 objectid, u64 bytenr, u64 orig_offset,
  835. u64 *start, u64 *end)
  836. {
  837. struct btrfs_file_extent_item *fi;
  838. struct btrfs_key key;
  839. u64 extent_end;
  840. if (slot < 0 || slot >= btrfs_header_nritems(leaf))
  841. return 0;
  842. btrfs_item_key_to_cpu(leaf, &key, slot);
  843. if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)
  844. return 0;
  845. fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
  846. if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||
  847. btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||
  848. btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||
  849. btrfs_file_extent_compression(leaf, fi) ||
  850. btrfs_file_extent_encryption(leaf, fi) ||
  851. btrfs_file_extent_other_encoding(leaf, fi))
  852. return 0;
  853. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  854. if ((*start && *start != key.offset) || (*end && *end != extent_end))
  855. return 0;
  856. *start = key.offset;
  857. *end = extent_end;
  858. return 1;
  859. }
  860. /*
  861. * Mark extent in the range start - end as written.
  862. *
  863. * This changes extent type from 'pre-allocated' to 'regular'. If only
  864. * part of extent is marked as written, the extent will be split into
  865. * two or three.
  866. */
  867. int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
  868. struct inode *inode, u64 start, u64 end)
  869. {
  870. struct btrfs_root *root = BTRFS_I(inode)->root;
  871. struct extent_buffer *leaf;
  872. struct btrfs_path *path;
  873. struct btrfs_file_extent_item *fi;
  874. struct btrfs_key key;
  875. struct btrfs_key new_key;
  876. u64 bytenr;
  877. u64 num_bytes;
  878. u64 extent_end;
  879. u64 orig_offset;
  880. u64 other_start;
  881. u64 other_end;
  882. u64 split;
  883. int del_nr = 0;
  884. int del_slot = 0;
  885. int recow;
  886. int ret;
  887. u64 ino = btrfs_ino(inode);
  888. path = btrfs_alloc_path();
  889. if (!path)
  890. return -ENOMEM;
  891. again:
  892. recow = 0;
  893. split = start;
  894. key.objectid = ino;
  895. key.type = BTRFS_EXTENT_DATA_KEY;
  896. key.offset = split;
  897. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  898. if (ret < 0)
  899. goto out;
  900. if (ret > 0 && path->slots[0] > 0)
  901. path->slots[0]--;
  902. leaf = path->nodes[0];
  903. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  904. BUG_ON(key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY);
  905. fi = btrfs_item_ptr(leaf, path->slots[0],
  906. struct btrfs_file_extent_item);
  907. BUG_ON(btrfs_file_extent_type(leaf, fi) !=
  908. BTRFS_FILE_EXTENT_PREALLOC);
  909. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  910. BUG_ON(key.offset > start || extent_end < end);
  911. bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  912. num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
  913. orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);
  914. memcpy(&new_key, &key, sizeof(new_key));
  915. if (start == key.offset && end < extent_end) {
  916. other_start = 0;
  917. other_end = start;
  918. if (extent_mergeable(leaf, path->slots[0] - 1,
  919. ino, bytenr, orig_offset,
  920. &other_start, &other_end)) {
  921. new_key.offset = end;
  922. btrfs_set_item_key_safe(trans, root, path, &new_key);
  923. fi = btrfs_item_ptr(leaf, path->slots[0],
  924. struct btrfs_file_extent_item);
  925. btrfs_set_file_extent_generation(leaf, fi,
  926. trans->transid);
  927. btrfs_set_file_extent_num_bytes(leaf, fi,
  928. extent_end - end);
  929. btrfs_set_file_extent_offset(leaf, fi,
  930. end - orig_offset);
  931. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  932. struct btrfs_file_extent_item);
  933. btrfs_set_file_extent_generation(leaf, fi,
  934. trans->transid);
  935. btrfs_set_file_extent_num_bytes(leaf, fi,
  936. end - other_start);
  937. btrfs_mark_buffer_dirty(leaf);
  938. goto out;
  939. }
  940. }
  941. if (start > key.offset && end == extent_end) {
  942. other_start = end;
  943. other_end = 0;
  944. if (extent_mergeable(leaf, path->slots[0] + 1,
  945. ino, bytenr, orig_offset,
  946. &other_start, &other_end)) {
  947. fi = btrfs_item_ptr(leaf, path->slots[0],
  948. struct btrfs_file_extent_item);
  949. btrfs_set_file_extent_num_bytes(leaf, fi,
  950. start - key.offset);
  951. btrfs_set_file_extent_generation(leaf, fi,
  952. trans->transid);
  953. path->slots[0]++;
  954. new_key.offset = start;
  955. btrfs_set_item_key_safe(trans, root, path, &new_key);
  956. fi = btrfs_item_ptr(leaf, path->slots[0],
  957. struct btrfs_file_extent_item);
  958. btrfs_set_file_extent_generation(leaf, fi,
  959. trans->transid);
  960. btrfs_set_file_extent_num_bytes(leaf, fi,
  961. other_end - start);
  962. btrfs_set_file_extent_offset(leaf, fi,
  963. start - orig_offset);
  964. btrfs_mark_buffer_dirty(leaf);
  965. goto out;
  966. }
  967. }
  968. while (start > key.offset || end < extent_end) {
  969. if (key.offset == start)
  970. split = end;
  971. new_key.offset = split;
  972. ret = btrfs_duplicate_item(trans, root, path, &new_key);
  973. if (ret == -EAGAIN) {
  974. btrfs_release_path(path);
  975. goto again;
  976. }
  977. if (ret < 0) {
  978. btrfs_abort_transaction(trans, root, ret);
  979. goto out;
  980. }
  981. leaf = path->nodes[0];
  982. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  983. struct btrfs_file_extent_item);
  984. btrfs_set_file_extent_generation(leaf, fi, trans->transid);
  985. btrfs_set_file_extent_num_bytes(leaf, fi,
  986. split - key.offset);
  987. fi = btrfs_item_ptr(leaf, path->slots[0],
  988. struct btrfs_file_extent_item);
  989. btrfs_set_file_extent_generation(leaf, fi, trans->transid);
  990. btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);
  991. btrfs_set_file_extent_num_bytes(leaf, fi,
  992. extent_end - split);
  993. btrfs_mark_buffer_dirty(leaf);
  994. ret = btrfs_inc_extent_ref(trans, root, bytenr, num_bytes, 0,
  995. root->root_key.objectid,
  996. ino, orig_offset, 0);
  997. BUG_ON(ret); /* -ENOMEM */
  998. if (split == start) {
  999. key.offset = start;
  1000. } else {
  1001. BUG_ON(start != key.offset);
  1002. path->slots[0]--;
  1003. extent_end = end;
  1004. }
  1005. recow = 1;
  1006. }
  1007. other_start = end;
  1008. other_end = 0;
  1009. if (extent_mergeable(leaf, path->slots[0] + 1,
  1010. ino, bytenr, orig_offset,
  1011. &other_start, &other_end)) {
  1012. if (recow) {
  1013. btrfs_release_path(path);
  1014. goto again;
  1015. }
  1016. extent_end = other_end;
  1017. del_slot = path->slots[0] + 1;
  1018. del_nr++;
  1019. ret = btrfs_free_extent(trans, root, bytenr, num_bytes,
  1020. 0, root->root_key.objectid,
  1021. ino, orig_offset, 0);
  1022. BUG_ON(ret); /* -ENOMEM */
  1023. }
  1024. other_start = 0;
  1025. other_end = start;
  1026. if (extent_mergeable(leaf, path->slots[0] - 1,
  1027. ino, bytenr, orig_offset,
  1028. &other_start, &other_end)) {
  1029. if (recow) {
  1030. btrfs_release_path(path);
  1031. goto again;
  1032. }
  1033. key.offset = other_start;
  1034. del_slot = path->slots[0];
  1035. del_nr++;
  1036. ret = btrfs_free_extent(trans, root, bytenr, num_bytes,
  1037. 0, root->root_key.objectid,
  1038. ino, orig_offset, 0);
  1039. BUG_ON(ret); /* -ENOMEM */
  1040. }
  1041. if (del_nr == 0) {
  1042. fi = btrfs_item_ptr(leaf, path->slots[0],
  1043. struct btrfs_file_extent_item);
  1044. btrfs_set_file_extent_type(leaf, fi,
  1045. BTRFS_FILE_EXTENT_REG);
  1046. btrfs_set_file_extent_generation(leaf, fi, trans->transid);
  1047. btrfs_mark_buffer_dirty(leaf);
  1048. } else {
  1049. fi = btrfs_item_ptr(leaf, del_slot - 1,
  1050. struct btrfs_file_extent_item);
  1051. btrfs_set_file_extent_type(leaf, fi,
  1052. BTRFS_FILE_EXTENT_REG);
  1053. btrfs_set_file_extent_generation(leaf, fi, trans->transid);
  1054. btrfs_set_file_extent_num_bytes(leaf, fi,
  1055. extent_end - key.offset);
  1056. btrfs_mark_buffer_dirty(leaf);
  1057. ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
  1058. if (ret < 0) {
  1059. btrfs_abort_transaction(trans, root, ret);
  1060. goto out;
  1061. }
  1062. }
  1063. out:
  1064. btrfs_free_path(path);
  1065. return 0;
  1066. }
  1067. /*
  1068. * on error we return an unlocked page and the error value
  1069. * on success we return a locked page and 0
  1070. */
  1071. static int prepare_uptodate_page(struct page *page, u64 pos,
  1072. bool force_uptodate)
  1073. {
  1074. int ret = 0;
  1075. if (((pos & (PAGE_CACHE_SIZE - 1)) || force_uptodate) &&
  1076. !PageUptodate(page)) {
  1077. ret = btrfs_readpage(NULL, page);
  1078. if (ret)
  1079. return ret;
  1080. lock_page(page);
  1081. if (!PageUptodate(page)) {
  1082. unlock_page(page);
  1083. return -EIO;
  1084. }
  1085. }
  1086. return 0;
  1087. }
  1088. /*
  1089. * this gets pages into the page cache and locks them down, it also properly
  1090. * waits for data=ordered extents to finish before allowing the pages to be
  1091. * modified.
  1092. */
  1093. static noinline int prepare_pages(struct btrfs_root *root, struct file *file,
  1094. struct page **pages, size_t num_pages,
  1095. loff_t pos, unsigned long first_index,
  1096. size_t write_bytes, bool force_uptodate)
  1097. {
  1098. struct extent_state *cached_state = NULL;
  1099. int i;
  1100. unsigned long index = pos >> PAGE_CACHE_SHIFT;
  1101. struct inode *inode = fdentry(file)->d_inode;
  1102. gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping);
  1103. int err = 0;
  1104. int faili = 0;
  1105. u64 start_pos;
  1106. u64 last_pos;
  1107. start_pos = pos & ~((u64)root->sectorsize - 1);
  1108. last_pos = ((u64)index + num_pages) << PAGE_CACHE_SHIFT;
  1109. again:
  1110. for (i = 0; i < num_pages; i++) {
  1111. pages[i] = find_or_create_page(inode->i_mapping, index + i,
  1112. mask | __GFP_WRITE);
  1113. if (!pages[i]) {
  1114. faili = i - 1;
  1115. err = -ENOMEM;
  1116. goto fail;
  1117. }
  1118. if (i == 0)
  1119. err = prepare_uptodate_page(pages[i], pos,
  1120. force_uptodate);
  1121. if (i == num_pages - 1)
  1122. err = prepare_uptodate_page(pages[i],
  1123. pos + write_bytes, false);
  1124. if (err) {
  1125. page_cache_release(pages[i]);
  1126. faili = i - 1;
  1127. goto fail;
  1128. }
  1129. wait_on_page_writeback(pages[i]);
  1130. }
  1131. err = 0;
  1132. if (start_pos < inode->i_size) {
  1133. struct btrfs_ordered_extent *ordered;
  1134. lock_extent_bits(&BTRFS_I(inode)->io_tree,
  1135. start_pos, last_pos - 1, 0, &cached_state);
  1136. ordered = btrfs_lookup_first_ordered_extent(inode,
  1137. last_pos - 1);
  1138. if (ordered &&
  1139. ordered->file_offset + ordered->len > start_pos &&
  1140. ordered->file_offset < last_pos) {
  1141. btrfs_put_ordered_extent(ordered);
  1142. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  1143. start_pos, last_pos - 1,
  1144. &cached_state, GFP_NOFS);
  1145. for (i = 0; i < num_pages; i++) {
  1146. unlock_page(pages[i]);
  1147. page_cache_release(pages[i]);
  1148. }
  1149. btrfs_wait_ordered_range(inode, start_pos,
  1150. last_pos - start_pos);
  1151. goto again;
  1152. }
  1153. if (ordered)
  1154. btrfs_put_ordered_extent(ordered);
  1155. clear_extent_bit(&BTRFS_I(inode)->io_tree, start_pos,
  1156. last_pos - 1, EXTENT_DIRTY | EXTENT_DELALLOC |
  1157. EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
  1158. 0, 0, &cached_state, GFP_NOFS);
  1159. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  1160. start_pos, last_pos - 1, &cached_state,
  1161. GFP_NOFS);
  1162. }
  1163. for (i = 0; i < num_pages; i++) {
  1164. if (clear_page_dirty_for_io(pages[i]))
  1165. account_page_redirty(pages[i]);
  1166. set_page_extent_mapped(pages[i]);
  1167. WARN_ON(!PageLocked(pages[i]));
  1168. }
  1169. return 0;
  1170. fail:
  1171. while (faili >= 0) {
  1172. unlock_page(pages[faili]);
  1173. page_cache_release(pages[faili]);
  1174. faili--;
  1175. }
  1176. return err;
  1177. }
  1178. static noinline ssize_t __btrfs_buffered_write(struct file *file,
  1179. struct iov_iter *i,
  1180. loff_t pos)
  1181. {
  1182. struct inode *inode = fdentry(file)->d_inode;
  1183. struct btrfs_root *root = BTRFS_I(inode)->root;
  1184. struct page **pages = NULL;
  1185. unsigned long first_index;
  1186. size_t num_written = 0;
  1187. int nrptrs;
  1188. int ret = 0;
  1189. bool force_page_uptodate = false;
  1190. nrptrs = min((iov_iter_count(i) + PAGE_CACHE_SIZE - 1) /
  1191. PAGE_CACHE_SIZE, PAGE_CACHE_SIZE /
  1192. (sizeof(struct page *)));
  1193. nrptrs = min(nrptrs, current->nr_dirtied_pause - current->nr_dirtied);
  1194. nrptrs = max(nrptrs, 8);
  1195. pages = kmalloc(nrptrs * sizeof(struct page *), GFP_KERNEL);
  1196. if (!pages)
  1197. return -ENOMEM;
  1198. first_index = pos >> PAGE_CACHE_SHIFT;
  1199. while (iov_iter_count(i) > 0) {
  1200. size_t offset = pos & (PAGE_CACHE_SIZE - 1);
  1201. size_t write_bytes = min(iov_iter_count(i),
  1202. nrptrs * (size_t)PAGE_CACHE_SIZE -
  1203. offset);
  1204. size_t num_pages = (write_bytes + offset +
  1205. PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1206. size_t dirty_pages;
  1207. size_t copied;
  1208. WARN_ON(num_pages > nrptrs);
  1209. /*
  1210. * Fault pages before locking them in prepare_pages
  1211. * to avoid recursive lock
  1212. */
  1213. if (unlikely(iov_iter_fault_in_readable(i, write_bytes))) {
  1214. ret = -EFAULT;
  1215. break;
  1216. }
  1217. ret = btrfs_delalloc_reserve_space(inode,
  1218. num_pages << PAGE_CACHE_SHIFT);
  1219. if (ret)
  1220. break;
  1221. /*
  1222. * This is going to setup the pages array with the number of
  1223. * pages we want, so we don't really need to worry about the
  1224. * contents of pages from loop to loop
  1225. */
  1226. ret = prepare_pages(root, file, pages, num_pages,
  1227. pos, first_index, write_bytes,
  1228. force_page_uptodate);
  1229. if (ret) {
  1230. btrfs_delalloc_release_space(inode,
  1231. num_pages << PAGE_CACHE_SHIFT);
  1232. break;
  1233. }
  1234. copied = btrfs_copy_from_user(pos, num_pages,
  1235. write_bytes, pages, i);
  1236. /*
  1237. * if we have trouble faulting in the pages, fall
  1238. * back to one page at a time
  1239. */
  1240. if (copied < write_bytes)
  1241. nrptrs = 1;
  1242. if (copied == 0) {
  1243. force_page_uptodate = true;
  1244. dirty_pages = 0;
  1245. } else {
  1246. force_page_uptodate = false;
  1247. dirty_pages = (copied + offset +
  1248. PAGE_CACHE_SIZE - 1) >>
  1249. PAGE_CACHE_SHIFT;
  1250. }
  1251. /*
  1252. * If we had a short copy we need to release the excess delaloc
  1253. * bytes we reserved. We need to increment outstanding_extents
  1254. * because btrfs_delalloc_release_space will decrement it, but
  1255. * we still have an outstanding extent for the chunk we actually
  1256. * managed to copy.
  1257. */
  1258. if (num_pages > dirty_pages) {
  1259. if (copied > 0) {
  1260. spin_lock(&BTRFS_I(inode)->lock);
  1261. BTRFS_I(inode)->outstanding_extents++;
  1262. spin_unlock(&BTRFS_I(inode)->lock);
  1263. }
  1264. btrfs_delalloc_release_space(inode,
  1265. (num_pages - dirty_pages) <<
  1266. PAGE_CACHE_SHIFT);
  1267. }
  1268. if (copied > 0) {
  1269. ret = btrfs_dirty_pages(root, inode, pages,
  1270. dirty_pages, pos, copied,
  1271. NULL);
  1272. if (ret) {
  1273. btrfs_delalloc_release_space(inode,
  1274. dirty_pages << PAGE_CACHE_SHIFT);
  1275. btrfs_drop_pages(pages, num_pages);
  1276. break;
  1277. }
  1278. }
  1279. btrfs_drop_pages(pages, num_pages);
  1280. cond_resched();
  1281. balance_dirty_pages_ratelimited_nr(inode->i_mapping,
  1282. dirty_pages);
  1283. if (dirty_pages < (root->leafsize >> PAGE_CACHE_SHIFT) + 1)
  1284. btrfs_btree_balance_dirty(root);
  1285. pos += copied;
  1286. num_written += copied;
  1287. }
  1288. kfree(pages);
  1289. return num_written ? num_written : ret;
  1290. }
  1291. static ssize_t __btrfs_direct_write(struct kiocb *iocb,
  1292. const struct iovec *iov,
  1293. unsigned long nr_segs, loff_t pos,
  1294. loff_t *ppos, size_t count, size_t ocount)
  1295. {
  1296. struct file *file = iocb->ki_filp;
  1297. struct iov_iter i;
  1298. ssize_t written;
  1299. ssize_t written_buffered;
  1300. loff_t endbyte;
  1301. int err;
  1302. written = generic_file_direct_write(iocb, iov, &nr_segs, pos, ppos,
  1303. count, ocount);
  1304. if (written < 0 || written == count)
  1305. return written;
  1306. pos += written;
  1307. count -= written;
  1308. iov_iter_init(&i, iov, nr_segs, count, written);
  1309. written_buffered = __btrfs_buffered_write(file, &i, pos);
  1310. if (written_buffered < 0) {
  1311. err = written_buffered;
  1312. goto out;
  1313. }
  1314. endbyte = pos + written_buffered - 1;
  1315. err = filemap_write_and_wait_range(file->f_mapping, pos, endbyte);
  1316. if (err)
  1317. goto out;
  1318. written += written_buffered;
  1319. *ppos = pos + written_buffered;
  1320. invalidate_mapping_pages(file->f_mapping, pos >> PAGE_CACHE_SHIFT,
  1321. endbyte >> PAGE_CACHE_SHIFT);
  1322. out:
  1323. return written ? written : err;
  1324. }
  1325. static void update_time_for_write(struct inode *inode)
  1326. {
  1327. struct timespec now;
  1328. if (IS_NOCMTIME(inode))
  1329. return;
  1330. now = current_fs_time(inode->i_sb);
  1331. if (!timespec_equal(&inode->i_mtime, &now))
  1332. inode->i_mtime = now;
  1333. if (!timespec_equal(&inode->i_ctime, &now))
  1334. inode->i_ctime = now;
  1335. if (IS_I_VERSION(inode))
  1336. inode_inc_iversion(inode);
  1337. }
  1338. static ssize_t btrfs_file_aio_write(struct kiocb *iocb,
  1339. const struct iovec *iov,
  1340. unsigned long nr_segs, loff_t pos)
  1341. {
  1342. struct file *file = iocb->ki_filp;
  1343. struct inode *inode = fdentry(file)->d_inode;
  1344. struct btrfs_root *root = BTRFS_I(inode)->root;
  1345. loff_t *ppos = &iocb->ki_pos;
  1346. u64 start_pos;
  1347. ssize_t num_written = 0;
  1348. ssize_t err = 0;
  1349. size_t count, ocount;
  1350. bool sync = (file->f_flags & O_DSYNC) || IS_SYNC(file->f_mapping->host);
  1351. sb_start_write(inode->i_sb);
  1352. mutex_lock(&inode->i_mutex);
  1353. err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ);
  1354. if (err) {
  1355. mutex_unlock(&inode->i_mutex);
  1356. goto out;
  1357. }
  1358. count = ocount;
  1359. current->backing_dev_info = inode->i_mapping->backing_dev_info;
  1360. err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
  1361. if (err) {
  1362. mutex_unlock(&inode->i_mutex);
  1363. goto out;
  1364. }
  1365. if (count == 0) {
  1366. mutex_unlock(&inode->i_mutex);
  1367. goto out;
  1368. }
  1369. err = file_remove_suid(file);
  1370. if (err) {
  1371. mutex_unlock(&inode->i_mutex);
  1372. goto out;
  1373. }
  1374. /*
  1375. * If BTRFS flips readonly due to some impossible error
  1376. * (fs_info->fs_state now has BTRFS_SUPER_FLAG_ERROR),
  1377. * although we have opened a file as writable, we have
  1378. * to stop this write operation to ensure FS consistency.
  1379. */
  1380. if (root->fs_info->fs_state & BTRFS_SUPER_FLAG_ERROR) {
  1381. mutex_unlock(&inode->i_mutex);
  1382. err = -EROFS;
  1383. goto out;
  1384. }
  1385. /*
  1386. * We reserve space for updating the inode when we reserve space for the
  1387. * extent we are going to write, so we will enospc out there. We don't
  1388. * need to start yet another transaction to update the inode as we will
  1389. * update the inode when we finish writing whatever data we write.
  1390. */
  1391. update_time_for_write(inode);
  1392. start_pos = round_down(pos, root->sectorsize);
  1393. if (start_pos > i_size_read(inode)) {
  1394. err = btrfs_cont_expand(inode, i_size_read(inode), start_pos);
  1395. if (err) {
  1396. mutex_unlock(&inode->i_mutex);
  1397. goto out;
  1398. }
  1399. }
  1400. if (sync)
  1401. atomic_inc(&BTRFS_I(inode)->sync_writers);
  1402. if (unlikely(file->f_flags & O_DIRECT)) {
  1403. num_written = __btrfs_direct_write(iocb, iov, nr_segs,
  1404. pos, ppos, count, ocount);
  1405. } else {
  1406. struct iov_iter i;
  1407. iov_iter_init(&i, iov, nr_segs, count, num_written);
  1408. num_written = __btrfs_buffered_write(file, &i, pos);
  1409. if (num_written > 0)
  1410. *ppos = pos + num_written;
  1411. }
  1412. mutex_unlock(&inode->i_mutex);
  1413. /*
  1414. * we want to make sure fsync finds this change
  1415. * but we haven't joined a transaction running right now.
  1416. *
  1417. * Later on, someone is sure to update the inode and get the
  1418. * real transid recorded.
  1419. *
  1420. * We set last_trans now to the fs_info generation + 1,
  1421. * this will either be one more than the running transaction
  1422. * or the generation used for the next transaction if there isn't
  1423. * one running right now.
  1424. *
  1425. * We also have to set last_sub_trans to the current log transid,
  1426. * otherwise subsequent syncs to a file that's been synced in this
  1427. * transaction will appear to have already occured.
  1428. */
  1429. BTRFS_I(inode)->last_trans = root->fs_info->generation + 1;
  1430. BTRFS_I(inode)->last_sub_trans = root->log_transid;
  1431. if (num_written > 0 || num_written == -EIOCBQUEUED) {
  1432. err = generic_write_sync(file, pos, num_written);
  1433. if (err < 0 && num_written > 0)
  1434. num_written = err;
  1435. }
  1436. if (sync)
  1437. atomic_dec(&BTRFS_I(inode)->sync_writers);
  1438. out:
  1439. sb_end_write(inode->i_sb);
  1440. current->backing_dev_info = NULL;
  1441. return num_written ? num_written : err;
  1442. }
  1443. int btrfs_release_file(struct inode *inode, struct file *filp)
  1444. {
  1445. /*
  1446. * ordered_data_close is set by settattr when we are about to truncate
  1447. * a file from a non-zero size to a zero size. This tries to
  1448. * flush down new bytes that may have been written if the
  1449. * application were using truncate to replace a file in place.
  1450. */
  1451. if (test_and_clear_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
  1452. &BTRFS_I(inode)->runtime_flags)) {
  1453. btrfs_add_ordered_operation(NULL, BTRFS_I(inode)->root, inode);
  1454. if (inode->i_size > BTRFS_ORDERED_OPERATIONS_FLUSH_LIMIT)
  1455. filemap_flush(inode->i_mapping);
  1456. }
  1457. if (filp->private_data)
  1458. btrfs_ioctl_trans_end(filp);
  1459. return 0;
  1460. }
  1461. /*
  1462. * fsync call for both files and directories. This logs the inode into
  1463. * the tree log instead of forcing full commits whenever possible.
  1464. *
  1465. * It needs to call filemap_fdatawait so that all ordered extent updates are
  1466. * in the metadata btree are up to date for copying to the log.
  1467. *
  1468. * It drops the inode mutex before doing the tree log commit. This is an
  1469. * important optimization for directories because holding the mutex prevents
  1470. * new operations on the dir while we write to disk.
  1471. */
  1472. int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
  1473. {
  1474. struct dentry *dentry = file->f_path.dentry;
  1475. struct inode *inode = dentry->d_inode;
  1476. struct btrfs_root *root = BTRFS_I(inode)->root;
  1477. int ret = 0;
  1478. struct btrfs_trans_handle *trans;
  1479. trace_btrfs_sync_file(file, datasync);
  1480. /*
  1481. * We write the dirty pages in the range and wait until they complete
  1482. * out of the ->i_mutex. If so, we can flush the dirty pages by
  1483. * multi-task, and make the performance up.
  1484. */
  1485. atomic_inc(&BTRFS_I(inode)->sync_writers);
  1486. ret = filemap_write_and_wait_range(inode->i_mapping, start, end);
  1487. atomic_dec(&BTRFS_I(inode)->sync_writers);
  1488. if (ret)
  1489. return ret;
  1490. mutex_lock(&inode->i_mutex);
  1491. /*
  1492. * We flush the dirty pages again to avoid some dirty pages in the
  1493. * range being left.
  1494. */
  1495. atomic_inc(&root->log_batch);
  1496. btrfs_wait_ordered_range(inode, start, end - start + 1);
  1497. atomic_inc(&root->log_batch);
  1498. /*
  1499. * check the transaction that last modified this inode
  1500. * and see if its already been committed
  1501. */
  1502. if (!BTRFS_I(inode)->last_trans) {
  1503. mutex_unlock(&inode->i_mutex);
  1504. goto out;
  1505. }
  1506. /*
  1507. * if the last transaction that changed this file was before
  1508. * the current transaction, we can bail out now without any
  1509. * syncing
  1510. */
  1511. smp_mb();
  1512. if (btrfs_inode_in_log(inode, root->fs_info->generation) ||
  1513. BTRFS_I(inode)->last_trans <=
  1514. root->fs_info->last_trans_committed) {
  1515. BTRFS_I(inode)->last_trans = 0;
  1516. /*
  1517. * We'v had everything committed since the last time we were
  1518. * modified so clear this flag in case it was set for whatever
  1519. * reason, it's no longer relevant.
  1520. */
  1521. clear_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
  1522. &BTRFS_I(inode)->runtime_flags);
  1523. mutex_unlock(&inode->i_mutex);
  1524. goto out;
  1525. }
  1526. /*
  1527. * ok we haven't committed the transaction yet, lets do a commit
  1528. */
  1529. if (file->private_data)
  1530. btrfs_ioctl_trans_end(file);
  1531. trans = btrfs_start_transaction(root, 0);
  1532. if (IS_ERR(trans)) {
  1533. ret = PTR_ERR(trans);
  1534. mutex_unlock(&inode->i_mutex);
  1535. goto out;
  1536. }
  1537. ret = btrfs_log_dentry_safe(trans, root, dentry);
  1538. if (ret < 0) {
  1539. mutex_unlock(&inode->i_mutex);
  1540. goto out;
  1541. }
  1542. /* we've logged all the items and now have a consistent
  1543. * version of the file in the log. It is possible that
  1544. * someone will come in and modify the file, but that's
  1545. * fine because the log is consistent on disk, and we
  1546. * have references to all of the file's extents
  1547. *
  1548. * It is possible that someone will come in and log the
  1549. * file again, but that will end up using the synchronization
  1550. * inside btrfs_sync_log to keep things safe.
  1551. */
  1552. mutex_unlock(&inode->i_mutex);
  1553. if (ret != BTRFS_NO_LOG_SYNC) {
  1554. if (ret > 0) {
  1555. ret = btrfs_commit_transaction(trans, root);
  1556. } else {
  1557. ret = btrfs_sync_log(trans, root);
  1558. if (ret == 0)
  1559. ret = btrfs_end_transaction(trans, root);
  1560. else
  1561. ret = btrfs_commit_transaction(trans, root);
  1562. }
  1563. } else {
  1564. ret = btrfs_end_transaction(trans, root);
  1565. }
  1566. out:
  1567. return ret > 0 ? -EIO : ret;
  1568. }
  1569. static const struct vm_operations_struct btrfs_file_vm_ops = {
  1570. .fault = filemap_fault,
  1571. .page_mkwrite = btrfs_page_mkwrite,
  1572. .remap_pages = generic_file_remap_pages,
  1573. };
  1574. static int btrfs_file_mmap(struct file *filp, struct vm_area_struct *vma)
  1575. {
  1576. struct address_space *mapping = filp->f_mapping;
  1577. if (!mapping->a_ops->readpage)
  1578. return -ENOEXEC;
  1579. file_accessed(filp);
  1580. vma->vm_ops = &btrfs_file_vm_ops;
  1581. return 0;
  1582. }
  1583. static int hole_mergeable(struct inode *inode, struct extent_buffer *leaf,
  1584. int slot, u64 start, u64 end)
  1585. {
  1586. struct btrfs_file_extent_item *fi;
  1587. struct btrfs_key key;
  1588. if (slot < 0 || slot >= btrfs_header_nritems(leaf))
  1589. return 0;
  1590. btrfs_item_key_to_cpu(leaf, &key, slot);
  1591. if (key.objectid != btrfs_ino(inode) ||
  1592. key.type != BTRFS_EXTENT_DATA_KEY)
  1593. return 0;
  1594. fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
  1595. if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
  1596. return 0;
  1597. if (btrfs_file_extent_disk_bytenr(leaf, fi))
  1598. return 0;
  1599. if (key.offset == end)
  1600. return 1;
  1601. if (key.offset + btrfs_file_extent_num_bytes(leaf, fi) == start)
  1602. return 1;
  1603. return 0;
  1604. }
  1605. static int fill_holes(struct btrfs_trans_handle *trans, struct inode *inode,
  1606. struct btrfs_path *path, u64 offset, u64 end)
  1607. {
  1608. struct btrfs_root *root = BTRFS_I(inode)->root;
  1609. struct extent_buffer *leaf;
  1610. struct btrfs_file_extent_item *fi;
  1611. struct extent_map *hole_em;
  1612. struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
  1613. struct btrfs_key key;
  1614. int ret;
  1615. key.objectid = btrfs_ino(inode);
  1616. key.type = BTRFS_EXTENT_DATA_KEY;
  1617. key.offset = offset;
  1618. ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
  1619. if (ret < 0)
  1620. return ret;
  1621. BUG_ON(!ret);
  1622. leaf = path->nodes[0];
  1623. if (hole_mergeable(inode, leaf, path->slots[0]-1, offset, end)) {
  1624. u64 num_bytes;
  1625. path->slots[0]--;
  1626. fi = btrfs_item_ptr(leaf, path->slots[0],
  1627. struct btrfs_file_extent_item);
  1628. num_bytes = btrfs_file_extent_num_bytes(leaf, fi) +
  1629. end - offset;
  1630. btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
  1631. btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
  1632. btrfs_set_file_extent_offset(leaf, fi, 0);
  1633. btrfs_mark_buffer_dirty(leaf);
  1634. goto out;
  1635. }
  1636. if (hole_mergeable(inode, leaf, path->slots[0]+1, offset, end)) {
  1637. u64 num_bytes;
  1638. path->slots[0]++;
  1639. key.offset = offset;
  1640. btrfs_set_item_key_safe(trans, root, path, &key);
  1641. fi = btrfs_item_ptr(leaf, path->slots[0],
  1642. struct btrfs_file_extent_item);
  1643. num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + end -
  1644. offset;
  1645. btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
  1646. btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
  1647. btrfs_set_file_extent_offset(leaf, fi, 0);
  1648. btrfs_mark_buffer_dirty(leaf);
  1649. goto out;
  1650. }
  1651. btrfs_release_path(path);
  1652. ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode), offset,
  1653. 0, 0, end - offset, 0, end - offset,
  1654. 0, 0, 0);
  1655. if (ret)
  1656. return ret;
  1657. out:
  1658. btrfs_release_path(path);
  1659. hole_em = alloc_extent_map();
  1660. if (!hole_em) {
  1661. btrfs_drop_extent_cache(inode, offset, end - 1, 0);
  1662. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
  1663. &BTRFS_I(inode)->runtime_flags);
  1664. } else {
  1665. hole_em->start = offset;
  1666. hole_em->len = end - offset;
  1667. hole_em->orig_start = offset;
  1668. hole_em->block_start = EXTENT_MAP_HOLE;
  1669. hole_em->block_len = 0;
  1670. hole_em->orig_block_len = 0;
  1671. hole_em->bdev = root->fs_info->fs_devices->latest_bdev;
  1672. hole_em->compress_type = BTRFS_COMPRESS_NONE;
  1673. hole_em->generation = trans->transid;
  1674. do {
  1675. btrfs_drop_extent_cache(inode, offset, end - 1, 0);
  1676. write_lock(&em_tree->lock);
  1677. ret = add_extent_mapping(em_tree, hole_em);
  1678. if (!ret)
  1679. list_move(&hole_em->list,
  1680. &em_tree->modified_extents);
  1681. write_unlock(&em_tree->lock);
  1682. } while (ret == -EEXIST);
  1683. free_extent_map(hole_em);
  1684. if (ret)
  1685. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
  1686. &BTRFS_I(inode)->runtime_flags);
  1687. }
  1688. return 0;
  1689. }
  1690. static int btrfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
  1691. {
  1692. struct btrfs_root *root = BTRFS_I(inode)->root;
  1693. struct extent_state *cached_state = NULL;
  1694. struct btrfs_path *path;
  1695. struct btrfs_block_rsv *rsv;
  1696. struct btrfs_trans_handle *trans;
  1697. u64 lockstart = round_up(offset, BTRFS_I(inode)->root->sectorsize);
  1698. u64 lockend = round_down(offset + len,
  1699. BTRFS_I(inode)->root->sectorsize) - 1;
  1700. u64 cur_offset = lockstart;
  1701. u64 min_size = btrfs_calc_trunc_metadata_size(root, 1);
  1702. u64 drop_end;
  1703. int ret = 0;
  1704. int err = 0;
  1705. bool same_page = ((offset >> PAGE_CACHE_SHIFT) ==
  1706. ((offset + len - 1) >> PAGE_CACHE_SHIFT));
  1707. btrfs_wait_ordered_range(inode, offset, len);
  1708. mutex_lock(&inode->i_mutex);
  1709. /*
  1710. * We needn't truncate any page which is beyond the end of the file
  1711. * because we are sure there is no data there.
  1712. */
  1713. /*
  1714. * Only do this if we are in the same page and we aren't doing the
  1715. * entire page.
  1716. */
  1717. if (same_page && len < PAGE_CACHE_SIZE) {
  1718. if (offset < round_up(inode->i_size, PAGE_CACHE_SIZE))
  1719. ret = btrfs_truncate_page(inode, offset, len, 0);
  1720. mutex_unlock(&inode->i_mutex);
  1721. return ret;
  1722. }
  1723. /* zero back part of the first page */
  1724. if (offset < round_up(inode->i_size, PAGE_CACHE_SIZE)) {
  1725. ret = btrfs_truncate_page(inode, offset, 0, 0);
  1726. if (ret) {
  1727. mutex_unlock(&inode->i_mutex);
  1728. return ret;
  1729. }
  1730. }
  1731. /* zero the front end of the last page */
  1732. if (offset + len < round_up(inode->i_size, PAGE_CACHE_SIZE)) {
  1733. ret = btrfs_truncate_page(inode, offset + len, 0, 1);
  1734. if (ret) {
  1735. mutex_unlock(&inode->i_mutex);
  1736. return ret;
  1737. }
  1738. }
  1739. if (lockend < lockstart) {
  1740. mutex_unlock(&inode->i_mutex);
  1741. return 0;
  1742. }
  1743. while (1) {
  1744. struct btrfs_ordered_extent *ordered;
  1745. truncate_pagecache_range(inode, lockstart, lockend);
  1746. lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  1747. 0, &cached_state);
  1748. ordered = btrfs_lookup_first_ordered_extent(inode, lockend);
  1749. /*
  1750. * We need to make sure we have no ordered extents in this range
  1751. * and nobody raced in and read a page in this range, if we did
  1752. * we need to try again.
  1753. */
  1754. if ((!ordered ||
  1755. (ordered->file_offset + ordered->len < lockstart ||
  1756. ordered->file_offset > lockend)) &&
  1757. !test_range_bit(&BTRFS_I(inode)->io_tree, lockstart,
  1758. lockend, EXTENT_UPTODATE, 0,
  1759. cached_state)) {
  1760. if (ordered)
  1761. btrfs_put_ordered_extent(ordered);
  1762. break;
  1763. }
  1764. if (ordered)
  1765. btrfs_put_ordered_extent(ordered);
  1766. unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
  1767. lockend, &cached_state, GFP_NOFS);
  1768. btrfs_wait_ordered_range(inode, lockstart,
  1769. lockend - lockstart + 1);
  1770. }
  1771. path = btrfs_alloc_path();
  1772. if (!path) {
  1773. ret = -ENOMEM;
  1774. goto out;
  1775. }
  1776. rsv = btrfs_alloc_block_rsv(root, BTRFS_BLOCK_RSV_TEMP);
  1777. if (!rsv) {
  1778. ret = -ENOMEM;
  1779. goto out_free;
  1780. }
  1781. rsv->size = btrfs_calc_trunc_metadata_size(root, 1);
  1782. rsv->failfast = 1;
  1783. /*
  1784. * 1 - update the inode
  1785. * 1 - removing the extents in the range
  1786. * 1 - adding the hole extent
  1787. */
  1788. trans = btrfs_start_transaction(root, 3);
  1789. if (IS_ERR(trans)) {
  1790. err = PTR_ERR(trans);
  1791. goto out_free;
  1792. }
  1793. ret = btrfs_block_rsv_migrate(&root->fs_info->trans_block_rsv, rsv,
  1794. min_size);
  1795. BUG_ON(ret);
  1796. trans->block_rsv = rsv;
  1797. while (cur_offset < lockend) {
  1798. ret = __btrfs_drop_extents(trans, root, inode, path,
  1799. cur_offset, lockend + 1,
  1800. &drop_end, 1);
  1801. if (ret != -ENOSPC)
  1802. break;
  1803. trans->block_rsv = &root->fs_info->trans_block_rsv;
  1804. ret = fill_holes(trans, inode, path, cur_offset, drop_end);
  1805. if (ret) {
  1806. err = ret;
  1807. break;
  1808. }
  1809. cur_offset = drop_end;
  1810. ret = btrfs_update_inode(trans, root, inode);
  1811. if (ret) {
  1812. err = ret;
  1813. break;
  1814. }
  1815. btrfs_end_transaction(trans, root);
  1816. btrfs_btree_balance_dirty(root);
  1817. trans = btrfs_start_transaction(root, 3);
  1818. if (IS_ERR(trans)) {
  1819. ret = PTR_ERR(trans);
  1820. trans = NULL;
  1821. break;
  1822. }
  1823. ret = btrfs_block_rsv_migrate(&root->fs_info->trans_block_rsv,
  1824. rsv, min_size);
  1825. BUG_ON(ret); /* shouldn't happen */
  1826. trans->block_rsv = rsv;
  1827. }
  1828. if (ret) {
  1829. err = ret;
  1830. goto out_trans;
  1831. }
  1832. trans->block_rsv = &root->fs_info->trans_block_rsv;
  1833. ret = fill_holes(trans, inode, path, cur_offset, drop_end);
  1834. if (ret) {
  1835. err = ret;
  1836. goto out_trans;
  1837. }
  1838. out_trans:
  1839. if (!trans)
  1840. goto out_free;
  1841. inode_inc_iversion(inode);
  1842. inode->i_mtime = inode->i_ctime = CURRENT_TIME;
  1843. trans->block_rsv = &root->fs_info->trans_block_rsv;
  1844. ret = btrfs_update_inode(trans, root, inode);
  1845. btrfs_end_transaction(trans, root);
  1846. btrfs_btree_balance_dirty(root);
  1847. out_free:
  1848. btrfs_free_path(path);
  1849. btrfs_free_block_rsv(root, rsv);
  1850. out:
  1851. unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  1852. &cached_state, GFP_NOFS);
  1853. mutex_unlock(&inode->i_mutex);
  1854. if (ret && !err)
  1855. err = ret;
  1856. return err;
  1857. }
  1858. static long btrfs_fallocate(struct file *file, int mode,
  1859. loff_t offset, loff_t len)
  1860. {
  1861. struct inode *inode = file->f_path.dentry->d_inode;
  1862. struct extent_state *cached_state = NULL;
  1863. u64 cur_offset;
  1864. u64 last_byte;
  1865. u64 alloc_start;
  1866. u64 alloc_end;
  1867. u64 alloc_hint = 0;
  1868. u64 locked_end;
  1869. struct extent_map *em;
  1870. int blocksize = BTRFS_I(inode)->root->sectorsize;
  1871. int ret;
  1872. alloc_start = round_down(offset, blocksize);
  1873. alloc_end = round_up(offset + len, blocksize);
  1874. /* Make sure we aren't being give some crap mode */
  1875. if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
  1876. return -EOPNOTSUPP;
  1877. if (mode & FALLOC_FL_PUNCH_HOLE)
  1878. return btrfs_punch_hole(inode, offset, len);
  1879. /*
  1880. * Make sure we have enough space before we do the
  1881. * allocation.
  1882. */
  1883. ret = btrfs_check_data_free_space(inode, alloc_end - alloc_start);
  1884. if (ret)
  1885. return ret;
  1886. /*
  1887. * wait for ordered IO before we have any locks. We'll loop again
  1888. * below with the locks held.
  1889. */
  1890. btrfs_wait_ordered_range(inode, alloc_start, alloc_end - alloc_start);
  1891. mutex_lock(&inode->i_mutex);
  1892. ret = inode_newsize_ok(inode, alloc_end);
  1893. if (ret)
  1894. goto out;
  1895. if (alloc_start > inode->i_size) {
  1896. ret = btrfs_cont_expand(inode, i_size_read(inode),
  1897. alloc_start);
  1898. if (ret)
  1899. goto out;
  1900. }
  1901. locked_end = alloc_end - 1;
  1902. while (1) {
  1903. struct btrfs_ordered_extent *ordered;
  1904. /* the extent lock is ordered inside the running
  1905. * transaction
  1906. */
  1907. lock_extent_bits(&BTRFS_I(inode)->io_tree, alloc_start,
  1908. locked_end, 0, &cached_state);
  1909. ordered = btrfs_lookup_first_ordered_extent(inode,
  1910. alloc_end - 1);
  1911. if (ordered &&
  1912. ordered->file_offset + ordered->len > alloc_start &&
  1913. ordered->file_offset < alloc_end) {
  1914. btrfs_put_ordered_extent(ordered);
  1915. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  1916. alloc_start, locked_end,
  1917. &cached_state, GFP_NOFS);
  1918. /*
  1919. * we can't wait on the range with the transaction
  1920. * running or with the extent lock held
  1921. */
  1922. btrfs_wait_ordered_range(inode, alloc_start,
  1923. alloc_end - alloc_start);
  1924. } else {
  1925. if (ordered)
  1926. btrfs_put_ordered_extent(ordered);
  1927. break;
  1928. }
  1929. }
  1930. cur_offset = alloc_start;
  1931. while (1) {
  1932. u64 actual_end;
  1933. em = btrfs_get_extent(inode, NULL, 0, cur_offset,
  1934. alloc_end - cur_offset, 0);
  1935. if (IS_ERR_OR_NULL(em)) {
  1936. if (!em)
  1937. ret = -ENOMEM;
  1938. else
  1939. ret = PTR_ERR(em);
  1940. break;
  1941. }
  1942. last_byte = min(extent_map_end(em), alloc_end);
  1943. actual_end = min_t(u64, extent_map_end(em), offset + len);
  1944. last_byte = ALIGN(last_byte, blocksize);
  1945. if (em->block_start == EXTENT_MAP_HOLE ||
  1946. (cur_offset >= inode->i_size &&
  1947. !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
  1948. ret = btrfs_prealloc_file_range(inode, mode, cur_offset,
  1949. last_byte - cur_offset,
  1950. 1 << inode->i_blkbits,
  1951. offset + len,
  1952. &alloc_hint);
  1953. if (ret < 0) {
  1954. free_extent_map(em);
  1955. break;
  1956. }
  1957. } else if (actual_end > inode->i_size &&
  1958. !(mode & FALLOC_FL_KEEP_SIZE)) {
  1959. /*
  1960. * We didn't need to allocate any more space, but we
  1961. * still extended the size of the file so we need to
  1962. * update i_size.
  1963. */
  1964. inode->i_ctime = CURRENT_TIME;
  1965. i_size_write(inode, actual_end);
  1966. btrfs_ordered_update_i_size(inode, actual_end, NULL);
  1967. }
  1968. free_extent_map(em);
  1969. cur_offset = last_byte;
  1970. if (cur_offset >= alloc_end) {
  1971. ret = 0;
  1972. break;
  1973. }
  1974. }
  1975. unlock_extent_cached(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
  1976. &cached_state, GFP_NOFS);
  1977. out:
  1978. mutex_unlock(&inode->i_mutex);
  1979. /* Let go of our reservation. */
  1980. btrfs_free_reserved_data_space(inode, alloc_end - alloc_start);
  1981. return ret;
  1982. }
  1983. static int find_desired_extent(struct inode *inode, loff_t *offset, int origin)
  1984. {
  1985. struct btrfs_root *root = BTRFS_I(inode)->root;
  1986. struct extent_map *em;
  1987. struct extent_state *cached_state = NULL;
  1988. u64 lockstart = *offset;
  1989. u64 lockend = i_size_read(inode);
  1990. u64 start = *offset;
  1991. u64 orig_start = *offset;
  1992. u64 len = i_size_read(inode);
  1993. u64 last_end = 0;
  1994. int ret = 0;
  1995. lockend = max_t(u64, root->sectorsize, lockend);
  1996. if (lockend <= lockstart)
  1997. lockend = lockstart + root->sectorsize;
  1998. lockend--;
  1999. len = lockend - lockstart + 1;
  2000. len = max_t(u64, len, root->sectorsize);
  2001. if (inode->i_size == 0)
  2002. return -ENXIO;
  2003. lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend, 0,
  2004. &cached_state);
  2005. /*
  2006. * Delalloc is such a pain. If we have a hole and we have pending
  2007. * delalloc for a portion of the hole we will get back a hole that
  2008. * exists for the entire range since it hasn't been actually written
  2009. * yet. So to take care of this case we need to look for an extent just
  2010. * before the position we want in case there is outstanding delalloc
  2011. * going on here.
  2012. */
  2013. if (origin == SEEK_HOLE && start != 0) {
  2014. if (start <= root->sectorsize)
  2015. em = btrfs_get_extent_fiemap(inode, NULL, 0, 0,
  2016. root->sectorsize, 0);
  2017. else
  2018. em = btrfs_get_extent_fiemap(inode, NULL, 0,
  2019. start - root->sectorsize,
  2020. root->sectorsize, 0);
  2021. if (IS_ERR(em)) {
  2022. ret = PTR_ERR(em);
  2023. goto out;
  2024. }
  2025. last_end = em->start + em->len;
  2026. if (em->block_start == EXTENT_MAP_DELALLOC)
  2027. last_end = min_t(u64, last_end, inode->i_size);
  2028. free_extent_map(em);
  2029. }
  2030. while (1) {
  2031. em = btrfs_get_extent_fiemap(inode, NULL, 0, start, len, 0);
  2032. if (IS_ERR(em)) {
  2033. ret = PTR_ERR(em);
  2034. break;
  2035. }
  2036. if (em->block_start == EXTENT_MAP_HOLE) {
  2037. if (test_bit(EXTENT_FLAG_VACANCY, &em->flags)) {
  2038. if (last_end <= orig_start) {
  2039. free_extent_map(em);
  2040. ret = -ENXIO;
  2041. break;
  2042. }
  2043. }
  2044. if (origin == SEEK_HOLE) {
  2045. *offset = start;
  2046. free_extent_map(em);
  2047. break;
  2048. }
  2049. } else {
  2050. if (origin == SEEK_DATA) {
  2051. if (em->block_start == EXTENT_MAP_DELALLOC) {
  2052. if (start >= inode->i_size) {
  2053. free_extent_map(em);
  2054. ret = -ENXIO;
  2055. break;
  2056. }
  2057. }
  2058. if (!test_bit(EXTENT_FLAG_PREALLOC,
  2059. &em->flags)) {
  2060. *offset = start;
  2061. free_extent_map(em);
  2062. break;
  2063. }
  2064. }
  2065. }
  2066. start = em->start + em->len;
  2067. last_end = em->start + em->len;
  2068. if (em->block_start == EXTENT_MAP_DELALLOC)
  2069. last_end = min_t(u64, last_end, inode->i_size);
  2070. if (test_bit(EXTENT_FLAG_VACANCY, &em->flags)) {
  2071. free_extent_map(em);
  2072. ret = -ENXIO;
  2073. break;
  2074. }
  2075. free_extent_map(em);
  2076. cond_resched();
  2077. }
  2078. if (!ret)
  2079. *offset = min(*offset, inode->i_size);
  2080. out:
  2081. unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  2082. &cached_state, GFP_NOFS);
  2083. return ret;
  2084. }
  2085. static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int origin)
  2086. {
  2087. struct inode *inode = file->f_mapping->host;
  2088. int ret;
  2089. mutex_lock(&inode->i_mutex);
  2090. switch (origin) {
  2091. case SEEK_END:
  2092. case SEEK_CUR:
  2093. offset = generic_file_llseek(file, offset, origin);
  2094. goto out;
  2095. case SEEK_DATA:
  2096. case SEEK_HOLE:
  2097. if (offset >= i_size_read(inode)) {
  2098. mutex_unlock(&inode->i_mutex);
  2099. return -ENXIO;
  2100. }
  2101. ret = find_desired_extent(inode, &offset, origin);
  2102. if (ret) {
  2103. mutex_unlock(&inode->i_mutex);
  2104. return ret;
  2105. }
  2106. }
  2107. if (offset < 0 && !(file->f_mode & FMODE_UNSIGNED_OFFSET)) {
  2108. offset = -EINVAL;
  2109. goto out;
  2110. }
  2111. if (offset > inode->i_sb->s_maxbytes) {
  2112. offset = -EINVAL;
  2113. goto out;
  2114. }
  2115. /* Special lock needed here? */
  2116. if (offset != file->f_pos) {
  2117. file->f_pos = offset;
  2118. file->f_version = 0;
  2119. }
  2120. out:
  2121. mutex_unlock(&inode->i_mutex);
  2122. return offset;
  2123. }
  2124. const struct file_operations btrfs_file_operations = {
  2125. .llseek = btrfs_file_llseek,
  2126. .read = do_sync_read,
  2127. .write = do_sync_write,
  2128. .aio_read = generic_file_aio_read,
  2129. .splice_read = generic_file_splice_read,
  2130. .aio_write = btrfs_file_aio_write,
  2131. .mmap = btrfs_file_mmap,
  2132. .open = generic_file_open,
  2133. .release = btrfs_release_file,
  2134. .fsync = btrfs_sync_file,
  2135. .fallocate = btrfs_fallocate,
  2136. .unlocked_ioctl = btrfs_ioctl,
  2137. #ifdef CONFIG_COMPAT
  2138. .compat_ioctl = btrfs_ioctl,
  2139. #endif
  2140. };
  2141. void btrfs_auto_defrag_exit(void)
  2142. {
  2143. if (btrfs_inode_defrag_cachep)
  2144. kmem_cache_destroy(btrfs_inode_defrag_cachep);
  2145. }
  2146. int btrfs_auto_defrag_init(void)
  2147. {
  2148. btrfs_inode_defrag_cachep = kmem_cache_create("btrfs_inode_defrag",
  2149. sizeof(struct inode_defrag), 0,
  2150. SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD,
  2151. NULL);
  2152. if (!btrfs_inode_defrag_cachep)
  2153. return -ENOMEM;
  2154. return 0;
  2155. }