file.c 47 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875
  1. /*
  2. * Copyright (C) 2007 Oracle. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public
  6. * License v2 as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. * General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public
  14. * License along with this program; if not, write to the
  15. * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16. * Boston, MA 021110-1307, USA.
  17. */
  18. #include <linux/fs.h>
  19. #include <linux/pagemap.h>
  20. #include <linux/highmem.h>
  21. #include <linux/time.h>
  22. #include <linux/init.h>
  23. #include <linux/string.h>
  24. #include <linux/backing-dev.h>
  25. #include <linux/mpage.h>
  26. #include <linux/falloc.h>
  27. #include <linux/swap.h>
  28. #include <linux/writeback.h>
  29. #include <linux/statfs.h>
  30. #include <linux/compat.h>
  31. #include <linux/slab.h>
  32. #include "ctree.h"
  33. #include "disk-io.h"
  34. #include "transaction.h"
  35. #include "btrfs_inode.h"
  36. #include "ioctl.h"
  37. #include "print-tree.h"
  38. #include "tree-log.h"
  39. #include "locking.h"
  40. #include "compat.h"
  41. /*
  42. * when auto defrag is enabled we
  43. * queue up these defrag structs to remember which
  44. * inodes need defragging passes
  45. */
  46. struct inode_defrag {
  47. struct rb_node rb_node;
  48. /* objectid */
  49. u64 ino;
  50. /*
  51. * transid where the defrag was added, we search for
  52. * extents newer than this
  53. */
  54. u64 transid;
  55. /* root objectid */
  56. u64 root;
  57. /* last offset we were able to defrag */
  58. u64 last_offset;
  59. /* if we've wrapped around back to zero once already */
  60. int cycled;
  61. };
  62. /* pop a record for an inode into the defrag tree. The lock
  63. * must be held already
  64. *
  65. * If you're inserting a record for an older transid than an
  66. * existing record, the transid already in the tree is lowered
  67. *
  68. * If an existing record is found the defrag item you
  69. * pass in is freed
  70. */
  71. static void __btrfs_add_inode_defrag(struct inode *inode,
  72. struct inode_defrag *defrag)
  73. {
  74. struct btrfs_root *root = BTRFS_I(inode)->root;
  75. struct inode_defrag *entry;
  76. struct rb_node **p;
  77. struct rb_node *parent = NULL;
  78. p = &root->fs_info->defrag_inodes.rb_node;
  79. while (*p) {
  80. parent = *p;
  81. entry = rb_entry(parent, struct inode_defrag, rb_node);
  82. if (defrag->ino < entry->ino)
  83. p = &parent->rb_left;
  84. else if (defrag->ino > entry->ino)
  85. p = &parent->rb_right;
  86. else {
  87. /* if we're reinserting an entry for
  88. * an old defrag run, make sure to
  89. * lower the transid of our existing record
  90. */
  91. if (defrag->transid < entry->transid)
  92. entry->transid = defrag->transid;
  93. if (defrag->last_offset > entry->last_offset)
  94. entry->last_offset = defrag->last_offset;
  95. goto exists;
  96. }
  97. }
  98. BTRFS_I(inode)->in_defrag = 1;
  99. rb_link_node(&defrag->rb_node, parent, p);
  100. rb_insert_color(&defrag->rb_node, &root->fs_info->defrag_inodes);
  101. return;
  102. exists:
  103. kfree(defrag);
  104. return;
  105. }
  106. /*
  107. * insert a defrag record for this inode if auto defrag is
  108. * enabled
  109. */
  110. int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
  111. struct inode *inode)
  112. {
  113. struct btrfs_root *root = BTRFS_I(inode)->root;
  114. struct inode_defrag *defrag;
  115. u64 transid;
  116. if (!btrfs_test_opt(root, AUTO_DEFRAG))
  117. return 0;
  118. if (btrfs_fs_closing(root->fs_info))
  119. return 0;
  120. if (BTRFS_I(inode)->in_defrag)
  121. return 0;
  122. if (trans)
  123. transid = trans->transid;
  124. else
  125. transid = BTRFS_I(inode)->root->last_trans;
  126. defrag = kzalloc(sizeof(*defrag), GFP_NOFS);
  127. if (!defrag)
  128. return -ENOMEM;
  129. defrag->ino = btrfs_ino(inode);
  130. defrag->transid = transid;
  131. defrag->root = root->root_key.objectid;
  132. spin_lock(&root->fs_info->defrag_inodes_lock);
  133. if (!BTRFS_I(inode)->in_defrag)
  134. __btrfs_add_inode_defrag(inode, defrag);
  135. else
  136. kfree(defrag);
  137. spin_unlock(&root->fs_info->defrag_inodes_lock);
  138. return 0;
  139. }
  140. /*
  141. * must be called with the defrag_inodes lock held
  142. */
  143. struct inode_defrag *btrfs_find_defrag_inode(struct btrfs_fs_info *info, u64 ino,
  144. struct rb_node **next)
  145. {
  146. struct inode_defrag *entry = NULL;
  147. struct rb_node *p;
  148. struct rb_node *parent = NULL;
  149. p = info->defrag_inodes.rb_node;
  150. while (p) {
  151. parent = p;
  152. entry = rb_entry(parent, struct inode_defrag, rb_node);
  153. if (ino < entry->ino)
  154. p = parent->rb_left;
  155. else if (ino > entry->ino)
  156. p = parent->rb_right;
  157. else
  158. return entry;
  159. }
  160. if (next) {
  161. while (parent && ino > entry->ino) {
  162. parent = rb_next(parent);
  163. entry = rb_entry(parent, struct inode_defrag, rb_node);
  164. }
  165. *next = parent;
  166. }
  167. return NULL;
  168. }
  169. /*
  170. * run through the list of inodes in the FS that need
  171. * defragging
  172. */
  173. int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info)
  174. {
  175. struct inode_defrag *defrag;
  176. struct btrfs_root *inode_root;
  177. struct inode *inode;
  178. struct rb_node *n;
  179. struct btrfs_key key;
  180. struct btrfs_ioctl_defrag_range_args range;
  181. u64 first_ino = 0;
  182. int num_defrag;
  183. int defrag_batch = 1024;
  184. memset(&range, 0, sizeof(range));
  185. range.len = (u64)-1;
  186. atomic_inc(&fs_info->defrag_running);
  187. spin_lock(&fs_info->defrag_inodes_lock);
  188. while(1) {
  189. n = NULL;
  190. /* find an inode to defrag */
  191. defrag = btrfs_find_defrag_inode(fs_info, first_ino, &n);
  192. if (!defrag) {
  193. if (n)
  194. defrag = rb_entry(n, struct inode_defrag, rb_node);
  195. else if (first_ino) {
  196. first_ino = 0;
  197. continue;
  198. } else {
  199. break;
  200. }
  201. }
  202. /* remove it from the rbtree */
  203. first_ino = defrag->ino + 1;
  204. rb_erase(&defrag->rb_node, &fs_info->defrag_inodes);
  205. if (btrfs_fs_closing(fs_info))
  206. goto next_free;
  207. spin_unlock(&fs_info->defrag_inodes_lock);
  208. /* get the inode */
  209. key.objectid = defrag->root;
  210. btrfs_set_key_type(&key, BTRFS_ROOT_ITEM_KEY);
  211. key.offset = (u64)-1;
  212. inode_root = btrfs_read_fs_root_no_name(fs_info, &key);
  213. if (IS_ERR(inode_root))
  214. goto next;
  215. key.objectid = defrag->ino;
  216. btrfs_set_key_type(&key, BTRFS_INODE_ITEM_KEY);
  217. key.offset = 0;
  218. inode = btrfs_iget(fs_info->sb, &key, inode_root, NULL);
  219. if (IS_ERR(inode))
  220. goto next;
  221. /* do a chunk of defrag */
  222. BTRFS_I(inode)->in_defrag = 0;
  223. range.start = defrag->last_offset;
  224. num_defrag = btrfs_defrag_file(inode, NULL, &range, defrag->transid,
  225. defrag_batch);
  226. /*
  227. * if we filled the whole defrag batch, there
  228. * must be more work to do. Queue this defrag
  229. * again
  230. */
  231. if (num_defrag == defrag_batch) {
  232. defrag->last_offset = range.start;
  233. __btrfs_add_inode_defrag(inode, defrag);
  234. /*
  235. * we don't want to kfree defrag, we added it back to
  236. * the rbtree
  237. */
  238. defrag = NULL;
  239. } else if (defrag->last_offset && !defrag->cycled) {
  240. /*
  241. * we didn't fill our defrag batch, but
  242. * we didn't start at zero. Make sure we loop
  243. * around to the start of the file.
  244. */
  245. defrag->last_offset = 0;
  246. defrag->cycled = 1;
  247. __btrfs_add_inode_defrag(inode, defrag);
  248. defrag = NULL;
  249. }
  250. iput(inode);
  251. next:
  252. spin_lock(&fs_info->defrag_inodes_lock);
  253. next_free:
  254. kfree(defrag);
  255. }
  256. spin_unlock(&fs_info->defrag_inodes_lock);
  257. atomic_dec(&fs_info->defrag_running);
  258. /*
  259. * during unmount, we use the transaction_wait queue to
  260. * wait for the defragger to stop
  261. */
  262. wake_up(&fs_info->transaction_wait);
  263. return 0;
  264. }
  265. /* simple helper to fault in pages and copy. This should go away
  266. * and be replaced with calls into generic code.
  267. */
  268. static noinline int btrfs_copy_from_user(loff_t pos, int num_pages,
  269. size_t write_bytes,
  270. struct page **prepared_pages,
  271. struct iov_iter *i)
  272. {
  273. size_t copied = 0;
  274. size_t total_copied = 0;
  275. int pg = 0;
  276. int offset = pos & (PAGE_CACHE_SIZE - 1);
  277. while (write_bytes > 0) {
  278. size_t count = min_t(size_t,
  279. PAGE_CACHE_SIZE - offset, write_bytes);
  280. struct page *page = prepared_pages[pg];
  281. /*
  282. * Copy data from userspace to the current page
  283. *
  284. * Disable pagefault to avoid recursive lock since
  285. * the pages are already locked
  286. */
  287. pagefault_disable();
  288. copied = iov_iter_copy_from_user_atomic(page, i, offset, count);
  289. pagefault_enable();
  290. /* Flush processor's dcache for this page */
  291. flush_dcache_page(page);
  292. /*
  293. * if we get a partial write, we can end up with
  294. * partially up to date pages. These add
  295. * a lot of complexity, so make sure they don't
  296. * happen by forcing this copy to be retried.
  297. *
  298. * The rest of the btrfs_file_write code will fall
  299. * back to page at a time copies after we return 0.
  300. */
  301. if (!PageUptodate(page) && copied < count)
  302. copied = 0;
  303. iov_iter_advance(i, copied);
  304. write_bytes -= copied;
  305. total_copied += copied;
  306. /* Return to btrfs_file_aio_write to fault page */
  307. if (unlikely(copied == 0))
  308. break;
  309. if (unlikely(copied < PAGE_CACHE_SIZE - offset)) {
  310. offset += copied;
  311. } else {
  312. pg++;
  313. offset = 0;
  314. }
  315. }
  316. return total_copied;
  317. }
  318. /*
  319. * unlocks pages after btrfs_file_write is done with them
  320. */
  321. void btrfs_drop_pages(struct page **pages, size_t num_pages)
  322. {
  323. size_t i;
  324. for (i = 0; i < num_pages; i++) {
  325. /* page checked is some magic around finding pages that
  326. * have been modified without going through btrfs_set_page_dirty
  327. * clear it here
  328. */
  329. ClearPageChecked(pages[i]);
  330. unlock_page(pages[i]);
  331. mark_page_accessed(pages[i]);
  332. page_cache_release(pages[i]);
  333. }
  334. }
  335. /*
  336. * after copy_from_user, pages need to be dirtied and we need to make
  337. * sure holes are created between the current EOF and the start of
  338. * any next extents (if required).
  339. *
  340. * this also makes the decision about creating an inline extent vs
  341. * doing real data extents, marking pages dirty and delalloc as required.
  342. */
  343. int btrfs_dirty_pages(struct btrfs_root *root, struct inode *inode,
  344. struct page **pages, size_t num_pages,
  345. loff_t pos, size_t write_bytes,
  346. struct extent_state **cached)
  347. {
  348. int err = 0;
  349. int i;
  350. u64 num_bytes;
  351. u64 start_pos;
  352. u64 end_of_last_block;
  353. u64 end_pos = pos + write_bytes;
  354. loff_t isize = i_size_read(inode);
  355. start_pos = pos & ~((u64)root->sectorsize - 1);
  356. num_bytes = (write_bytes + pos - start_pos +
  357. root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
  358. end_of_last_block = start_pos + num_bytes - 1;
  359. err = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block,
  360. cached);
  361. if (err)
  362. return err;
  363. for (i = 0; i < num_pages; i++) {
  364. struct page *p = pages[i];
  365. SetPageUptodate(p);
  366. ClearPageChecked(p);
  367. set_page_dirty(p);
  368. }
  369. /*
  370. * we've only changed i_size in ram, and we haven't updated
  371. * the disk i_size. There is no need to log the inode
  372. * at this time.
  373. */
  374. if (end_pos > isize)
  375. i_size_write(inode, end_pos);
  376. return 0;
  377. }
  378. /*
  379. * this drops all the extents in the cache that intersect the range
  380. * [start, end]. Existing extents are split as required.
  381. */
  382. int btrfs_drop_extent_cache(struct inode *inode, u64 start, u64 end,
  383. int skip_pinned)
  384. {
  385. struct extent_map *em;
  386. struct extent_map *split = NULL;
  387. struct extent_map *split2 = NULL;
  388. struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
  389. u64 len = end - start + 1;
  390. int ret;
  391. int testend = 1;
  392. unsigned long flags;
  393. int compressed = 0;
  394. WARN_ON(end < start);
  395. if (end == (u64)-1) {
  396. len = (u64)-1;
  397. testend = 0;
  398. }
  399. while (1) {
  400. if (!split)
  401. split = alloc_extent_map();
  402. if (!split2)
  403. split2 = alloc_extent_map();
  404. BUG_ON(!split || !split2);
  405. write_lock(&em_tree->lock);
  406. em = lookup_extent_mapping(em_tree, start, len);
  407. if (!em) {
  408. write_unlock(&em_tree->lock);
  409. break;
  410. }
  411. flags = em->flags;
  412. if (skip_pinned && test_bit(EXTENT_FLAG_PINNED, &em->flags)) {
  413. if (testend && em->start + em->len >= start + len) {
  414. free_extent_map(em);
  415. write_unlock(&em_tree->lock);
  416. break;
  417. }
  418. start = em->start + em->len;
  419. if (testend)
  420. len = start + len - (em->start + em->len);
  421. free_extent_map(em);
  422. write_unlock(&em_tree->lock);
  423. continue;
  424. }
  425. compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
  426. clear_bit(EXTENT_FLAG_PINNED, &em->flags);
  427. remove_extent_mapping(em_tree, em);
  428. if (em->block_start < EXTENT_MAP_LAST_BYTE &&
  429. em->start < start) {
  430. split->start = em->start;
  431. split->len = start - em->start;
  432. split->orig_start = em->orig_start;
  433. split->block_start = em->block_start;
  434. if (compressed)
  435. split->block_len = em->block_len;
  436. else
  437. split->block_len = split->len;
  438. split->bdev = em->bdev;
  439. split->flags = flags;
  440. split->compress_type = em->compress_type;
  441. ret = add_extent_mapping(em_tree, split);
  442. BUG_ON(ret);
  443. free_extent_map(split);
  444. split = split2;
  445. split2 = NULL;
  446. }
  447. if (em->block_start < EXTENT_MAP_LAST_BYTE &&
  448. testend && em->start + em->len > start + len) {
  449. u64 diff = start + len - em->start;
  450. split->start = start + len;
  451. split->len = em->start + em->len - (start + len);
  452. split->bdev = em->bdev;
  453. split->flags = flags;
  454. split->compress_type = em->compress_type;
  455. if (compressed) {
  456. split->block_len = em->block_len;
  457. split->block_start = em->block_start;
  458. split->orig_start = em->orig_start;
  459. } else {
  460. split->block_len = split->len;
  461. split->block_start = em->block_start + diff;
  462. split->orig_start = split->start;
  463. }
  464. ret = add_extent_mapping(em_tree, split);
  465. BUG_ON(ret);
  466. free_extent_map(split);
  467. split = NULL;
  468. }
  469. write_unlock(&em_tree->lock);
  470. /* once for us */
  471. free_extent_map(em);
  472. /* once for the tree*/
  473. free_extent_map(em);
  474. }
  475. if (split)
  476. free_extent_map(split);
  477. if (split2)
  478. free_extent_map(split2);
  479. return 0;
  480. }
  481. /*
  482. * this is very complex, but the basic idea is to drop all extents
  483. * in the range start - end. hint_block is filled in with a block number
  484. * that would be a good hint to the block allocator for this file.
  485. *
  486. * If an extent intersects the range but is not entirely inside the range
  487. * it is either truncated or split. Anything entirely inside the range
  488. * is deleted from the tree.
  489. */
  490. int btrfs_drop_extents(struct btrfs_trans_handle *trans, struct inode *inode,
  491. u64 start, u64 end, u64 *hint_byte, int drop_cache)
  492. {
  493. struct btrfs_root *root = BTRFS_I(inode)->root;
  494. struct extent_buffer *leaf;
  495. struct btrfs_file_extent_item *fi;
  496. struct btrfs_path *path;
  497. struct btrfs_key key;
  498. struct btrfs_key new_key;
  499. u64 ino = btrfs_ino(inode);
  500. u64 search_start = start;
  501. u64 disk_bytenr = 0;
  502. u64 num_bytes = 0;
  503. u64 extent_offset = 0;
  504. u64 extent_end = 0;
  505. int del_nr = 0;
  506. int del_slot = 0;
  507. int extent_type;
  508. int recow;
  509. int ret;
  510. if (drop_cache)
  511. btrfs_drop_extent_cache(inode, start, end - 1, 0);
  512. path = btrfs_alloc_path();
  513. if (!path)
  514. return -ENOMEM;
  515. while (1) {
  516. recow = 0;
  517. ret = btrfs_lookup_file_extent(trans, root, path, ino,
  518. search_start, -1);
  519. if (ret < 0)
  520. break;
  521. if (ret > 0 && path->slots[0] > 0 && search_start == start) {
  522. leaf = path->nodes[0];
  523. btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
  524. if (key.objectid == ino &&
  525. key.type == BTRFS_EXTENT_DATA_KEY)
  526. path->slots[0]--;
  527. }
  528. ret = 0;
  529. next_slot:
  530. leaf = path->nodes[0];
  531. if (path->slots[0] >= btrfs_header_nritems(leaf)) {
  532. BUG_ON(del_nr > 0);
  533. ret = btrfs_next_leaf(root, path);
  534. if (ret < 0)
  535. break;
  536. if (ret > 0) {
  537. ret = 0;
  538. break;
  539. }
  540. leaf = path->nodes[0];
  541. recow = 1;
  542. }
  543. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  544. if (key.objectid > ino ||
  545. key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= end)
  546. break;
  547. fi = btrfs_item_ptr(leaf, path->slots[0],
  548. struct btrfs_file_extent_item);
  549. extent_type = btrfs_file_extent_type(leaf, fi);
  550. if (extent_type == BTRFS_FILE_EXTENT_REG ||
  551. extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
  552. disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  553. num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
  554. extent_offset = btrfs_file_extent_offset(leaf, fi);
  555. extent_end = key.offset +
  556. btrfs_file_extent_num_bytes(leaf, fi);
  557. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  558. extent_end = key.offset +
  559. btrfs_file_extent_inline_len(leaf, fi);
  560. } else {
  561. WARN_ON(1);
  562. extent_end = search_start;
  563. }
  564. if (extent_end <= search_start) {
  565. path->slots[0]++;
  566. goto next_slot;
  567. }
  568. search_start = max(key.offset, start);
  569. if (recow) {
  570. btrfs_release_path(path);
  571. continue;
  572. }
  573. /*
  574. * | - range to drop - |
  575. * | -------- extent -------- |
  576. */
  577. if (start > key.offset && end < extent_end) {
  578. BUG_ON(del_nr > 0);
  579. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  580. memcpy(&new_key, &key, sizeof(new_key));
  581. new_key.offset = start;
  582. ret = btrfs_duplicate_item(trans, root, path,
  583. &new_key);
  584. if (ret == -EAGAIN) {
  585. btrfs_release_path(path);
  586. continue;
  587. }
  588. if (ret < 0)
  589. break;
  590. leaf = path->nodes[0];
  591. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  592. struct btrfs_file_extent_item);
  593. btrfs_set_file_extent_num_bytes(leaf, fi,
  594. start - key.offset);
  595. fi = btrfs_item_ptr(leaf, path->slots[0],
  596. struct btrfs_file_extent_item);
  597. extent_offset += start - key.offset;
  598. btrfs_set_file_extent_offset(leaf, fi, extent_offset);
  599. btrfs_set_file_extent_num_bytes(leaf, fi,
  600. extent_end - start);
  601. btrfs_mark_buffer_dirty(leaf);
  602. if (disk_bytenr > 0) {
  603. ret = btrfs_inc_extent_ref(trans, root,
  604. disk_bytenr, num_bytes, 0,
  605. root->root_key.objectid,
  606. new_key.objectid,
  607. start - extent_offset);
  608. BUG_ON(ret);
  609. *hint_byte = disk_bytenr;
  610. }
  611. key.offset = start;
  612. }
  613. /*
  614. * | ---- range to drop ----- |
  615. * | -------- extent -------- |
  616. */
  617. if (start <= key.offset && end < extent_end) {
  618. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  619. memcpy(&new_key, &key, sizeof(new_key));
  620. new_key.offset = end;
  621. btrfs_set_item_key_safe(trans, root, path, &new_key);
  622. extent_offset += end - key.offset;
  623. btrfs_set_file_extent_offset(leaf, fi, extent_offset);
  624. btrfs_set_file_extent_num_bytes(leaf, fi,
  625. extent_end - end);
  626. btrfs_mark_buffer_dirty(leaf);
  627. if (disk_bytenr > 0) {
  628. inode_sub_bytes(inode, end - key.offset);
  629. *hint_byte = disk_bytenr;
  630. }
  631. break;
  632. }
  633. search_start = extent_end;
  634. /*
  635. * | ---- range to drop ----- |
  636. * | -------- extent -------- |
  637. */
  638. if (start > key.offset && end >= extent_end) {
  639. BUG_ON(del_nr > 0);
  640. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  641. btrfs_set_file_extent_num_bytes(leaf, fi,
  642. start - key.offset);
  643. btrfs_mark_buffer_dirty(leaf);
  644. if (disk_bytenr > 0) {
  645. inode_sub_bytes(inode, extent_end - start);
  646. *hint_byte = disk_bytenr;
  647. }
  648. if (end == extent_end)
  649. break;
  650. path->slots[0]++;
  651. goto next_slot;
  652. }
  653. /*
  654. * | ---- range to drop ----- |
  655. * | ------ extent ------ |
  656. */
  657. if (start <= key.offset && end >= extent_end) {
  658. if (del_nr == 0) {
  659. del_slot = path->slots[0];
  660. del_nr = 1;
  661. } else {
  662. BUG_ON(del_slot + del_nr != path->slots[0]);
  663. del_nr++;
  664. }
  665. if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  666. inode_sub_bytes(inode,
  667. extent_end - key.offset);
  668. extent_end = ALIGN(extent_end,
  669. root->sectorsize);
  670. } else if (disk_bytenr > 0) {
  671. ret = btrfs_free_extent(trans, root,
  672. disk_bytenr, num_bytes, 0,
  673. root->root_key.objectid,
  674. key.objectid, key.offset -
  675. extent_offset);
  676. BUG_ON(ret);
  677. inode_sub_bytes(inode,
  678. extent_end - key.offset);
  679. *hint_byte = disk_bytenr;
  680. }
  681. if (end == extent_end)
  682. break;
  683. if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {
  684. path->slots[0]++;
  685. goto next_slot;
  686. }
  687. ret = btrfs_del_items(trans, root, path, del_slot,
  688. del_nr);
  689. BUG_ON(ret);
  690. del_nr = 0;
  691. del_slot = 0;
  692. btrfs_release_path(path);
  693. continue;
  694. }
  695. BUG_ON(1);
  696. }
  697. if (del_nr > 0) {
  698. ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
  699. BUG_ON(ret);
  700. }
  701. btrfs_free_path(path);
  702. return ret;
  703. }
  704. static int extent_mergeable(struct extent_buffer *leaf, int slot,
  705. u64 objectid, u64 bytenr, u64 orig_offset,
  706. u64 *start, u64 *end)
  707. {
  708. struct btrfs_file_extent_item *fi;
  709. struct btrfs_key key;
  710. u64 extent_end;
  711. if (slot < 0 || slot >= btrfs_header_nritems(leaf))
  712. return 0;
  713. btrfs_item_key_to_cpu(leaf, &key, slot);
  714. if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)
  715. return 0;
  716. fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
  717. if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||
  718. btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||
  719. btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||
  720. btrfs_file_extent_compression(leaf, fi) ||
  721. btrfs_file_extent_encryption(leaf, fi) ||
  722. btrfs_file_extent_other_encoding(leaf, fi))
  723. return 0;
  724. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  725. if ((*start && *start != key.offset) || (*end && *end != extent_end))
  726. return 0;
  727. *start = key.offset;
  728. *end = extent_end;
  729. return 1;
  730. }
  731. /*
  732. * Mark extent in the range start - end as written.
  733. *
  734. * This changes extent type from 'pre-allocated' to 'regular'. If only
  735. * part of extent is marked as written, the extent will be split into
  736. * two or three.
  737. */
  738. int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
  739. struct inode *inode, u64 start, u64 end)
  740. {
  741. struct btrfs_root *root = BTRFS_I(inode)->root;
  742. struct extent_buffer *leaf;
  743. struct btrfs_path *path;
  744. struct btrfs_file_extent_item *fi;
  745. struct btrfs_key key;
  746. struct btrfs_key new_key;
  747. u64 bytenr;
  748. u64 num_bytes;
  749. u64 extent_end;
  750. u64 orig_offset;
  751. u64 other_start;
  752. u64 other_end;
  753. u64 split;
  754. int del_nr = 0;
  755. int del_slot = 0;
  756. int recow;
  757. int ret;
  758. u64 ino = btrfs_ino(inode);
  759. btrfs_drop_extent_cache(inode, start, end - 1, 0);
  760. path = btrfs_alloc_path();
  761. if (!path)
  762. return -ENOMEM;
  763. again:
  764. recow = 0;
  765. split = start;
  766. key.objectid = ino;
  767. key.type = BTRFS_EXTENT_DATA_KEY;
  768. key.offset = split;
  769. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  770. if (ret < 0)
  771. goto out;
  772. if (ret > 0 && path->slots[0] > 0)
  773. path->slots[0]--;
  774. leaf = path->nodes[0];
  775. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  776. BUG_ON(key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY);
  777. fi = btrfs_item_ptr(leaf, path->slots[0],
  778. struct btrfs_file_extent_item);
  779. BUG_ON(btrfs_file_extent_type(leaf, fi) !=
  780. BTRFS_FILE_EXTENT_PREALLOC);
  781. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  782. BUG_ON(key.offset > start || extent_end < end);
  783. bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  784. num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
  785. orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);
  786. memcpy(&new_key, &key, sizeof(new_key));
  787. if (start == key.offset && end < extent_end) {
  788. other_start = 0;
  789. other_end = start;
  790. if (extent_mergeable(leaf, path->slots[0] - 1,
  791. ino, bytenr, orig_offset,
  792. &other_start, &other_end)) {
  793. new_key.offset = end;
  794. btrfs_set_item_key_safe(trans, root, path, &new_key);
  795. fi = btrfs_item_ptr(leaf, path->slots[0],
  796. struct btrfs_file_extent_item);
  797. btrfs_set_file_extent_num_bytes(leaf, fi,
  798. extent_end - end);
  799. btrfs_set_file_extent_offset(leaf, fi,
  800. end - orig_offset);
  801. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  802. struct btrfs_file_extent_item);
  803. btrfs_set_file_extent_num_bytes(leaf, fi,
  804. end - other_start);
  805. btrfs_mark_buffer_dirty(leaf);
  806. goto out;
  807. }
  808. }
  809. if (start > key.offset && end == extent_end) {
  810. other_start = end;
  811. other_end = 0;
  812. if (extent_mergeable(leaf, path->slots[0] + 1,
  813. ino, bytenr, orig_offset,
  814. &other_start, &other_end)) {
  815. fi = btrfs_item_ptr(leaf, path->slots[0],
  816. struct btrfs_file_extent_item);
  817. btrfs_set_file_extent_num_bytes(leaf, fi,
  818. start - key.offset);
  819. path->slots[0]++;
  820. new_key.offset = start;
  821. btrfs_set_item_key_safe(trans, root, path, &new_key);
  822. fi = btrfs_item_ptr(leaf, path->slots[0],
  823. struct btrfs_file_extent_item);
  824. btrfs_set_file_extent_num_bytes(leaf, fi,
  825. other_end - start);
  826. btrfs_set_file_extent_offset(leaf, fi,
  827. start - orig_offset);
  828. btrfs_mark_buffer_dirty(leaf);
  829. goto out;
  830. }
  831. }
  832. while (start > key.offset || end < extent_end) {
  833. if (key.offset == start)
  834. split = end;
  835. new_key.offset = split;
  836. ret = btrfs_duplicate_item(trans, root, path, &new_key);
  837. if (ret == -EAGAIN) {
  838. btrfs_release_path(path);
  839. goto again;
  840. }
  841. BUG_ON(ret < 0);
  842. leaf = path->nodes[0];
  843. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  844. struct btrfs_file_extent_item);
  845. btrfs_set_file_extent_num_bytes(leaf, fi,
  846. split - key.offset);
  847. fi = btrfs_item_ptr(leaf, path->slots[0],
  848. struct btrfs_file_extent_item);
  849. btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);
  850. btrfs_set_file_extent_num_bytes(leaf, fi,
  851. extent_end - split);
  852. btrfs_mark_buffer_dirty(leaf);
  853. ret = btrfs_inc_extent_ref(trans, root, bytenr, num_bytes, 0,
  854. root->root_key.objectid,
  855. ino, orig_offset);
  856. BUG_ON(ret);
  857. if (split == start) {
  858. key.offset = start;
  859. } else {
  860. BUG_ON(start != key.offset);
  861. path->slots[0]--;
  862. extent_end = end;
  863. }
  864. recow = 1;
  865. }
  866. other_start = end;
  867. other_end = 0;
  868. if (extent_mergeable(leaf, path->slots[0] + 1,
  869. ino, bytenr, orig_offset,
  870. &other_start, &other_end)) {
  871. if (recow) {
  872. btrfs_release_path(path);
  873. goto again;
  874. }
  875. extent_end = other_end;
  876. del_slot = path->slots[0] + 1;
  877. del_nr++;
  878. ret = btrfs_free_extent(trans, root, bytenr, num_bytes,
  879. 0, root->root_key.objectid,
  880. ino, orig_offset);
  881. BUG_ON(ret);
  882. }
  883. other_start = 0;
  884. other_end = start;
  885. if (extent_mergeable(leaf, path->slots[0] - 1,
  886. ino, bytenr, orig_offset,
  887. &other_start, &other_end)) {
  888. if (recow) {
  889. btrfs_release_path(path);
  890. goto again;
  891. }
  892. key.offset = other_start;
  893. del_slot = path->slots[0];
  894. del_nr++;
  895. ret = btrfs_free_extent(trans, root, bytenr, num_bytes,
  896. 0, root->root_key.objectid,
  897. ino, orig_offset);
  898. BUG_ON(ret);
  899. }
  900. if (del_nr == 0) {
  901. fi = btrfs_item_ptr(leaf, path->slots[0],
  902. struct btrfs_file_extent_item);
  903. btrfs_set_file_extent_type(leaf, fi,
  904. BTRFS_FILE_EXTENT_REG);
  905. btrfs_mark_buffer_dirty(leaf);
  906. } else {
  907. fi = btrfs_item_ptr(leaf, del_slot - 1,
  908. struct btrfs_file_extent_item);
  909. btrfs_set_file_extent_type(leaf, fi,
  910. BTRFS_FILE_EXTENT_REG);
  911. btrfs_set_file_extent_num_bytes(leaf, fi,
  912. extent_end - key.offset);
  913. btrfs_mark_buffer_dirty(leaf);
  914. ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
  915. BUG_ON(ret);
  916. }
  917. out:
  918. btrfs_free_path(path);
  919. return 0;
  920. }
  921. /*
  922. * on error we return an unlocked page and the error value
  923. * on success we return a locked page and 0
  924. */
  925. static int prepare_uptodate_page(struct page *page, u64 pos,
  926. bool force_uptodate)
  927. {
  928. int ret = 0;
  929. if (((pos & (PAGE_CACHE_SIZE - 1)) || force_uptodate) &&
  930. !PageUptodate(page)) {
  931. ret = btrfs_readpage(NULL, page);
  932. if (ret)
  933. return ret;
  934. lock_page(page);
  935. if (!PageUptodate(page)) {
  936. unlock_page(page);
  937. return -EIO;
  938. }
  939. }
  940. return 0;
  941. }
  942. /*
  943. * this gets pages into the page cache and locks them down, it also properly
  944. * waits for data=ordered extents to finish before allowing the pages to be
  945. * modified.
  946. */
  947. static noinline int prepare_pages(struct btrfs_root *root, struct file *file,
  948. struct page **pages, size_t num_pages,
  949. loff_t pos, unsigned long first_index,
  950. size_t write_bytes, bool force_uptodate)
  951. {
  952. struct extent_state *cached_state = NULL;
  953. int i;
  954. unsigned long index = pos >> PAGE_CACHE_SHIFT;
  955. struct inode *inode = fdentry(file)->d_inode;
  956. int err = 0;
  957. int faili = 0;
  958. u64 start_pos;
  959. u64 last_pos;
  960. start_pos = pos & ~((u64)root->sectorsize - 1);
  961. last_pos = ((u64)index + num_pages) << PAGE_CACHE_SHIFT;
  962. again:
  963. for (i = 0; i < num_pages; i++) {
  964. pages[i] = find_or_create_page(inode->i_mapping, index + i,
  965. GFP_NOFS);
  966. if (!pages[i]) {
  967. faili = i - 1;
  968. err = -ENOMEM;
  969. goto fail;
  970. }
  971. if (i == 0)
  972. err = prepare_uptodate_page(pages[i], pos,
  973. force_uptodate);
  974. if (i == num_pages - 1)
  975. err = prepare_uptodate_page(pages[i],
  976. pos + write_bytes, false);
  977. if (err) {
  978. page_cache_release(pages[i]);
  979. faili = i - 1;
  980. goto fail;
  981. }
  982. wait_on_page_writeback(pages[i]);
  983. }
  984. err = 0;
  985. if (start_pos < inode->i_size) {
  986. struct btrfs_ordered_extent *ordered;
  987. lock_extent_bits(&BTRFS_I(inode)->io_tree,
  988. start_pos, last_pos - 1, 0, &cached_state,
  989. GFP_NOFS);
  990. ordered = btrfs_lookup_first_ordered_extent(inode,
  991. last_pos - 1);
  992. if (ordered &&
  993. ordered->file_offset + ordered->len > start_pos &&
  994. ordered->file_offset < last_pos) {
  995. btrfs_put_ordered_extent(ordered);
  996. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  997. start_pos, last_pos - 1,
  998. &cached_state, GFP_NOFS);
  999. for (i = 0; i < num_pages; i++) {
  1000. unlock_page(pages[i]);
  1001. page_cache_release(pages[i]);
  1002. }
  1003. btrfs_wait_ordered_range(inode, start_pos,
  1004. last_pos - start_pos);
  1005. goto again;
  1006. }
  1007. if (ordered)
  1008. btrfs_put_ordered_extent(ordered);
  1009. clear_extent_bit(&BTRFS_I(inode)->io_tree, start_pos,
  1010. last_pos - 1, EXTENT_DIRTY | EXTENT_DELALLOC |
  1011. EXTENT_DO_ACCOUNTING, 0, 0, &cached_state,
  1012. GFP_NOFS);
  1013. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  1014. start_pos, last_pos - 1, &cached_state,
  1015. GFP_NOFS);
  1016. }
  1017. for (i = 0; i < num_pages; i++) {
  1018. clear_page_dirty_for_io(pages[i]);
  1019. set_page_extent_mapped(pages[i]);
  1020. WARN_ON(!PageLocked(pages[i]));
  1021. }
  1022. return 0;
  1023. fail:
  1024. while (faili >= 0) {
  1025. unlock_page(pages[faili]);
  1026. page_cache_release(pages[faili]);
  1027. faili--;
  1028. }
  1029. return err;
  1030. }
  1031. static noinline ssize_t __btrfs_buffered_write(struct file *file,
  1032. struct iov_iter *i,
  1033. loff_t pos)
  1034. {
  1035. struct inode *inode = fdentry(file)->d_inode;
  1036. struct btrfs_root *root = BTRFS_I(inode)->root;
  1037. struct page **pages = NULL;
  1038. unsigned long first_index;
  1039. size_t num_written = 0;
  1040. int nrptrs;
  1041. int ret = 0;
  1042. bool force_page_uptodate = false;
  1043. nrptrs = min((iov_iter_count(i) + PAGE_CACHE_SIZE - 1) /
  1044. PAGE_CACHE_SIZE, PAGE_CACHE_SIZE /
  1045. (sizeof(struct page *)));
  1046. pages = kmalloc(nrptrs * sizeof(struct page *), GFP_KERNEL);
  1047. if (!pages)
  1048. return -ENOMEM;
  1049. first_index = pos >> PAGE_CACHE_SHIFT;
  1050. while (iov_iter_count(i) > 0) {
  1051. size_t offset = pos & (PAGE_CACHE_SIZE - 1);
  1052. size_t write_bytes = min(iov_iter_count(i),
  1053. nrptrs * (size_t)PAGE_CACHE_SIZE -
  1054. offset);
  1055. size_t num_pages = (write_bytes + offset +
  1056. PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1057. size_t dirty_pages;
  1058. size_t copied;
  1059. WARN_ON(num_pages > nrptrs);
  1060. /*
  1061. * Fault pages before locking them in prepare_pages
  1062. * to avoid recursive lock
  1063. */
  1064. if (unlikely(iov_iter_fault_in_readable(i, write_bytes))) {
  1065. ret = -EFAULT;
  1066. break;
  1067. }
  1068. ret = btrfs_delalloc_reserve_space(inode,
  1069. num_pages << PAGE_CACHE_SHIFT);
  1070. if (ret)
  1071. break;
  1072. /*
  1073. * This is going to setup the pages array with the number of
  1074. * pages we want, so we don't really need to worry about the
  1075. * contents of pages from loop to loop
  1076. */
  1077. ret = prepare_pages(root, file, pages, num_pages,
  1078. pos, first_index, write_bytes,
  1079. force_page_uptodate);
  1080. if (ret) {
  1081. btrfs_delalloc_release_space(inode,
  1082. num_pages << PAGE_CACHE_SHIFT);
  1083. break;
  1084. }
  1085. copied = btrfs_copy_from_user(pos, num_pages,
  1086. write_bytes, pages, i);
  1087. /*
  1088. * if we have trouble faulting in the pages, fall
  1089. * back to one page at a time
  1090. */
  1091. if (copied < write_bytes)
  1092. nrptrs = 1;
  1093. if (copied == 0) {
  1094. force_page_uptodate = true;
  1095. dirty_pages = 0;
  1096. } else {
  1097. force_page_uptodate = false;
  1098. dirty_pages = (copied + offset +
  1099. PAGE_CACHE_SIZE - 1) >>
  1100. PAGE_CACHE_SHIFT;
  1101. }
  1102. /*
  1103. * If we had a short copy we need to release the excess delaloc
  1104. * bytes we reserved. We need to increment outstanding_extents
  1105. * because btrfs_delalloc_release_space will decrement it, but
  1106. * we still have an outstanding extent for the chunk we actually
  1107. * managed to copy.
  1108. */
  1109. if (num_pages > dirty_pages) {
  1110. if (copied > 0) {
  1111. spin_lock(&BTRFS_I(inode)->lock);
  1112. BTRFS_I(inode)->outstanding_extents++;
  1113. spin_unlock(&BTRFS_I(inode)->lock);
  1114. }
  1115. btrfs_delalloc_release_space(inode,
  1116. (num_pages - dirty_pages) <<
  1117. PAGE_CACHE_SHIFT);
  1118. }
  1119. if (copied > 0) {
  1120. ret = btrfs_dirty_pages(root, inode, pages,
  1121. dirty_pages, pos, copied,
  1122. NULL);
  1123. if (ret) {
  1124. btrfs_delalloc_release_space(inode,
  1125. dirty_pages << PAGE_CACHE_SHIFT);
  1126. btrfs_drop_pages(pages, num_pages);
  1127. break;
  1128. }
  1129. }
  1130. btrfs_drop_pages(pages, num_pages);
  1131. cond_resched();
  1132. balance_dirty_pages_ratelimited_nr(inode->i_mapping,
  1133. dirty_pages);
  1134. if (dirty_pages < (root->leafsize >> PAGE_CACHE_SHIFT) + 1)
  1135. btrfs_btree_balance_dirty(root, 1);
  1136. btrfs_throttle(root);
  1137. pos += copied;
  1138. num_written += copied;
  1139. }
  1140. kfree(pages);
  1141. return num_written ? num_written : ret;
  1142. }
  1143. static ssize_t __btrfs_direct_write(struct kiocb *iocb,
  1144. const struct iovec *iov,
  1145. unsigned long nr_segs, loff_t pos,
  1146. loff_t *ppos, size_t count, size_t ocount)
  1147. {
  1148. struct file *file = iocb->ki_filp;
  1149. struct inode *inode = fdentry(file)->d_inode;
  1150. struct iov_iter i;
  1151. ssize_t written;
  1152. ssize_t written_buffered;
  1153. loff_t endbyte;
  1154. int err;
  1155. written = generic_file_direct_write(iocb, iov, &nr_segs, pos, ppos,
  1156. count, ocount);
  1157. /*
  1158. * the generic O_DIRECT will update in-memory i_size after the
  1159. * DIOs are done. But our endio handlers that update the on
  1160. * disk i_size never update past the in memory i_size. So we
  1161. * need one more update here to catch any additions to the
  1162. * file
  1163. */
  1164. if (inode->i_size != BTRFS_I(inode)->disk_i_size) {
  1165. btrfs_ordered_update_i_size(inode, inode->i_size, NULL);
  1166. mark_inode_dirty(inode);
  1167. }
  1168. if (written < 0 || written == count)
  1169. return written;
  1170. pos += written;
  1171. count -= written;
  1172. iov_iter_init(&i, iov, nr_segs, count, written);
  1173. written_buffered = __btrfs_buffered_write(file, &i, pos);
  1174. if (written_buffered < 0) {
  1175. err = written_buffered;
  1176. goto out;
  1177. }
  1178. endbyte = pos + written_buffered - 1;
  1179. err = filemap_write_and_wait_range(file->f_mapping, pos, endbyte);
  1180. if (err)
  1181. goto out;
  1182. written += written_buffered;
  1183. *ppos = pos + written_buffered;
  1184. invalidate_mapping_pages(file->f_mapping, pos >> PAGE_CACHE_SHIFT,
  1185. endbyte >> PAGE_CACHE_SHIFT);
  1186. out:
  1187. return written ? written : err;
  1188. }
  1189. static ssize_t btrfs_file_aio_write(struct kiocb *iocb,
  1190. const struct iovec *iov,
  1191. unsigned long nr_segs, loff_t pos)
  1192. {
  1193. struct file *file = iocb->ki_filp;
  1194. struct inode *inode = fdentry(file)->d_inode;
  1195. struct btrfs_root *root = BTRFS_I(inode)->root;
  1196. loff_t *ppos = &iocb->ki_pos;
  1197. u64 start_pos;
  1198. ssize_t num_written = 0;
  1199. ssize_t err = 0;
  1200. size_t count, ocount;
  1201. vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE);
  1202. mutex_lock(&inode->i_mutex);
  1203. err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ);
  1204. if (err) {
  1205. mutex_unlock(&inode->i_mutex);
  1206. goto out;
  1207. }
  1208. count = ocount;
  1209. current->backing_dev_info = inode->i_mapping->backing_dev_info;
  1210. err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
  1211. if (err) {
  1212. mutex_unlock(&inode->i_mutex);
  1213. goto out;
  1214. }
  1215. if (count == 0) {
  1216. mutex_unlock(&inode->i_mutex);
  1217. goto out;
  1218. }
  1219. err = file_remove_suid(file);
  1220. if (err) {
  1221. mutex_unlock(&inode->i_mutex);
  1222. goto out;
  1223. }
  1224. /*
  1225. * If BTRFS flips readonly due to some impossible error
  1226. * (fs_info->fs_state now has BTRFS_SUPER_FLAG_ERROR),
  1227. * although we have opened a file as writable, we have
  1228. * to stop this write operation to ensure FS consistency.
  1229. */
  1230. if (root->fs_info->fs_state & BTRFS_SUPER_FLAG_ERROR) {
  1231. mutex_unlock(&inode->i_mutex);
  1232. err = -EROFS;
  1233. goto out;
  1234. }
  1235. file_update_time(file);
  1236. BTRFS_I(inode)->sequence++;
  1237. start_pos = round_down(pos, root->sectorsize);
  1238. if (start_pos > i_size_read(inode)) {
  1239. err = btrfs_cont_expand(inode, i_size_read(inode), start_pos);
  1240. if (err) {
  1241. mutex_unlock(&inode->i_mutex);
  1242. goto out;
  1243. }
  1244. }
  1245. if (unlikely(file->f_flags & O_DIRECT)) {
  1246. num_written = __btrfs_direct_write(iocb, iov, nr_segs,
  1247. pos, ppos, count, ocount);
  1248. } else {
  1249. struct iov_iter i;
  1250. iov_iter_init(&i, iov, nr_segs, count, num_written);
  1251. num_written = __btrfs_buffered_write(file, &i, pos);
  1252. if (num_written > 0)
  1253. *ppos = pos + num_written;
  1254. }
  1255. mutex_unlock(&inode->i_mutex);
  1256. /*
  1257. * we want to make sure fsync finds this change
  1258. * but we haven't joined a transaction running right now.
  1259. *
  1260. * Later on, someone is sure to update the inode and get the
  1261. * real transid recorded.
  1262. *
  1263. * We set last_trans now to the fs_info generation + 1,
  1264. * this will either be one more than the running transaction
  1265. * or the generation used for the next transaction if there isn't
  1266. * one running right now.
  1267. */
  1268. BTRFS_I(inode)->last_trans = root->fs_info->generation + 1;
  1269. if (num_written > 0 || num_written == -EIOCBQUEUED) {
  1270. err = generic_write_sync(file, pos, num_written);
  1271. if (err < 0 && num_written > 0)
  1272. num_written = err;
  1273. }
  1274. out:
  1275. current->backing_dev_info = NULL;
  1276. return num_written ? num_written : err;
  1277. }
  1278. int btrfs_release_file(struct inode *inode, struct file *filp)
  1279. {
  1280. /*
  1281. * ordered_data_close is set by settattr when we are about to truncate
  1282. * a file from a non-zero size to a zero size. This tries to
  1283. * flush down new bytes that may have been written if the
  1284. * application were using truncate to replace a file in place.
  1285. */
  1286. if (BTRFS_I(inode)->ordered_data_close) {
  1287. BTRFS_I(inode)->ordered_data_close = 0;
  1288. btrfs_add_ordered_operation(NULL, BTRFS_I(inode)->root, inode);
  1289. if (inode->i_size > BTRFS_ORDERED_OPERATIONS_FLUSH_LIMIT)
  1290. filemap_flush(inode->i_mapping);
  1291. }
  1292. if (filp->private_data)
  1293. btrfs_ioctl_trans_end(filp);
  1294. return 0;
  1295. }
  1296. /*
  1297. * fsync call for both files and directories. This logs the inode into
  1298. * the tree log instead of forcing full commits whenever possible.
  1299. *
  1300. * It needs to call filemap_fdatawait so that all ordered extent updates are
  1301. * in the metadata btree are up to date for copying to the log.
  1302. *
  1303. * It drops the inode mutex before doing the tree log commit. This is an
  1304. * important optimization for directories because holding the mutex prevents
  1305. * new operations on the dir while we write to disk.
  1306. */
  1307. int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
  1308. {
  1309. struct dentry *dentry = file->f_path.dentry;
  1310. struct inode *inode = dentry->d_inode;
  1311. struct btrfs_root *root = BTRFS_I(inode)->root;
  1312. int ret = 0;
  1313. struct btrfs_trans_handle *trans;
  1314. trace_btrfs_sync_file(file, datasync);
  1315. ret = filemap_write_and_wait_range(inode->i_mapping, start, end);
  1316. if (ret)
  1317. return ret;
  1318. mutex_lock(&inode->i_mutex);
  1319. /* we wait first, since the writeback may change the inode */
  1320. root->log_batch++;
  1321. btrfs_wait_ordered_range(inode, 0, (u64)-1);
  1322. root->log_batch++;
  1323. /*
  1324. * check the transaction that last modified this inode
  1325. * and see if its already been committed
  1326. */
  1327. if (!BTRFS_I(inode)->last_trans) {
  1328. mutex_unlock(&inode->i_mutex);
  1329. goto out;
  1330. }
  1331. /*
  1332. * if the last transaction that changed this file was before
  1333. * the current transaction, we can bail out now without any
  1334. * syncing
  1335. */
  1336. smp_mb();
  1337. if (BTRFS_I(inode)->last_trans <=
  1338. root->fs_info->last_trans_committed) {
  1339. BTRFS_I(inode)->last_trans = 0;
  1340. mutex_unlock(&inode->i_mutex);
  1341. goto out;
  1342. }
  1343. /*
  1344. * ok we haven't committed the transaction yet, lets do a commit
  1345. */
  1346. if (file->private_data)
  1347. btrfs_ioctl_trans_end(file);
  1348. trans = btrfs_start_transaction(root, 0);
  1349. if (IS_ERR(trans)) {
  1350. ret = PTR_ERR(trans);
  1351. mutex_unlock(&inode->i_mutex);
  1352. goto out;
  1353. }
  1354. ret = btrfs_log_dentry_safe(trans, root, dentry);
  1355. if (ret < 0) {
  1356. mutex_unlock(&inode->i_mutex);
  1357. goto out;
  1358. }
  1359. /* we've logged all the items and now have a consistent
  1360. * version of the file in the log. It is possible that
  1361. * someone will come in and modify the file, but that's
  1362. * fine because the log is consistent on disk, and we
  1363. * have references to all of the file's extents
  1364. *
  1365. * It is possible that someone will come in and log the
  1366. * file again, but that will end up using the synchronization
  1367. * inside btrfs_sync_log to keep things safe.
  1368. */
  1369. mutex_unlock(&inode->i_mutex);
  1370. if (ret != BTRFS_NO_LOG_SYNC) {
  1371. if (ret > 0) {
  1372. ret = btrfs_commit_transaction(trans, root);
  1373. } else {
  1374. ret = btrfs_sync_log(trans, root);
  1375. if (ret == 0)
  1376. ret = btrfs_end_transaction(trans, root);
  1377. else
  1378. ret = btrfs_commit_transaction(trans, root);
  1379. }
  1380. } else {
  1381. ret = btrfs_end_transaction(trans, root);
  1382. }
  1383. out:
  1384. return ret > 0 ? -EIO : ret;
  1385. }
  1386. static const struct vm_operations_struct btrfs_file_vm_ops = {
  1387. .fault = filemap_fault,
  1388. .page_mkwrite = btrfs_page_mkwrite,
  1389. };
  1390. static int btrfs_file_mmap(struct file *filp, struct vm_area_struct *vma)
  1391. {
  1392. struct address_space *mapping = filp->f_mapping;
  1393. if (!mapping->a_ops->readpage)
  1394. return -ENOEXEC;
  1395. file_accessed(filp);
  1396. vma->vm_ops = &btrfs_file_vm_ops;
  1397. vma->vm_flags |= VM_CAN_NONLINEAR;
  1398. return 0;
  1399. }
  1400. static long btrfs_fallocate(struct file *file, int mode,
  1401. loff_t offset, loff_t len)
  1402. {
  1403. struct inode *inode = file->f_path.dentry->d_inode;
  1404. struct extent_state *cached_state = NULL;
  1405. u64 cur_offset;
  1406. u64 last_byte;
  1407. u64 alloc_start;
  1408. u64 alloc_end;
  1409. u64 alloc_hint = 0;
  1410. u64 locked_end;
  1411. u64 mask = BTRFS_I(inode)->root->sectorsize - 1;
  1412. struct extent_map *em;
  1413. int ret;
  1414. alloc_start = offset & ~mask;
  1415. alloc_end = (offset + len + mask) & ~mask;
  1416. /* We only support the FALLOC_FL_KEEP_SIZE mode */
  1417. if (mode & ~FALLOC_FL_KEEP_SIZE)
  1418. return -EOPNOTSUPP;
  1419. /*
  1420. * wait for ordered IO before we have any locks. We'll loop again
  1421. * below with the locks held.
  1422. */
  1423. btrfs_wait_ordered_range(inode, alloc_start, alloc_end - alloc_start);
  1424. mutex_lock(&inode->i_mutex);
  1425. ret = inode_newsize_ok(inode, alloc_end);
  1426. if (ret)
  1427. goto out;
  1428. if (alloc_start > inode->i_size) {
  1429. ret = btrfs_cont_expand(inode, i_size_read(inode),
  1430. alloc_start);
  1431. if (ret)
  1432. goto out;
  1433. }
  1434. ret = btrfs_check_data_free_space(inode, alloc_end - alloc_start);
  1435. if (ret)
  1436. goto out;
  1437. locked_end = alloc_end - 1;
  1438. while (1) {
  1439. struct btrfs_ordered_extent *ordered;
  1440. /* the extent lock is ordered inside the running
  1441. * transaction
  1442. */
  1443. lock_extent_bits(&BTRFS_I(inode)->io_tree, alloc_start,
  1444. locked_end, 0, &cached_state, GFP_NOFS);
  1445. ordered = btrfs_lookup_first_ordered_extent(inode,
  1446. alloc_end - 1);
  1447. if (ordered &&
  1448. ordered->file_offset + ordered->len > alloc_start &&
  1449. ordered->file_offset < alloc_end) {
  1450. btrfs_put_ordered_extent(ordered);
  1451. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  1452. alloc_start, locked_end,
  1453. &cached_state, GFP_NOFS);
  1454. /*
  1455. * we can't wait on the range with the transaction
  1456. * running or with the extent lock held
  1457. */
  1458. btrfs_wait_ordered_range(inode, alloc_start,
  1459. alloc_end - alloc_start);
  1460. } else {
  1461. if (ordered)
  1462. btrfs_put_ordered_extent(ordered);
  1463. break;
  1464. }
  1465. }
  1466. cur_offset = alloc_start;
  1467. while (1) {
  1468. u64 actual_end;
  1469. em = btrfs_get_extent(inode, NULL, 0, cur_offset,
  1470. alloc_end - cur_offset, 0);
  1471. BUG_ON(IS_ERR_OR_NULL(em));
  1472. last_byte = min(extent_map_end(em), alloc_end);
  1473. actual_end = min_t(u64, extent_map_end(em), offset + len);
  1474. last_byte = (last_byte + mask) & ~mask;
  1475. if (em->block_start == EXTENT_MAP_HOLE ||
  1476. (cur_offset >= inode->i_size &&
  1477. !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
  1478. ret = btrfs_prealloc_file_range(inode, mode, cur_offset,
  1479. last_byte - cur_offset,
  1480. 1 << inode->i_blkbits,
  1481. offset + len,
  1482. &alloc_hint);
  1483. if (ret < 0) {
  1484. free_extent_map(em);
  1485. break;
  1486. }
  1487. } else if (actual_end > inode->i_size &&
  1488. !(mode & FALLOC_FL_KEEP_SIZE)) {
  1489. /*
  1490. * We didn't need to allocate any more space, but we
  1491. * still extended the size of the file so we need to
  1492. * update i_size.
  1493. */
  1494. inode->i_ctime = CURRENT_TIME;
  1495. i_size_write(inode, actual_end);
  1496. btrfs_ordered_update_i_size(inode, actual_end, NULL);
  1497. }
  1498. free_extent_map(em);
  1499. cur_offset = last_byte;
  1500. if (cur_offset >= alloc_end) {
  1501. ret = 0;
  1502. break;
  1503. }
  1504. }
  1505. unlock_extent_cached(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
  1506. &cached_state, GFP_NOFS);
  1507. btrfs_free_reserved_data_space(inode, alloc_end - alloc_start);
  1508. out:
  1509. mutex_unlock(&inode->i_mutex);
  1510. return ret;
  1511. }
  1512. static int find_desired_extent(struct inode *inode, loff_t *offset, int origin)
  1513. {
  1514. struct btrfs_root *root = BTRFS_I(inode)->root;
  1515. struct extent_map *em;
  1516. struct extent_state *cached_state = NULL;
  1517. u64 lockstart = *offset;
  1518. u64 lockend = i_size_read(inode);
  1519. u64 start = *offset;
  1520. u64 orig_start = *offset;
  1521. u64 len = i_size_read(inode);
  1522. u64 last_end = 0;
  1523. int ret = 0;
  1524. lockend = max_t(u64, root->sectorsize, lockend);
  1525. if (lockend <= lockstart)
  1526. lockend = lockstart + root->sectorsize;
  1527. len = lockend - lockstart + 1;
  1528. len = max_t(u64, len, root->sectorsize);
  1529. if (inode->i_size == 0)
  1530. return -ENXIO;
  1531. lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend, 0,
  1532. &cached_state, GFP_NOFS);
  1533. /*
  1534. * Delalloc is such a pain. If we have a hole and we have pending
  1535. * delalloc for a portion of the hole we will get back a hole that
  1536. * exists for the entire range since it hasn't been actually written
  1537. * yet. So to take care of this case we need to look for an extent just
  1538. * before the position we want in case there is outstanding delalloc
  1539. * going on here.
  1540. */
  1541. if (origin == SEEK_HOLE && start != 0) {
  1542. if (start <= root->sectorsize)
  1543. em = btrfs_get_extent_fiemap(inode, NULL, 0, 0,
  1544. root->sectorsize, 0);
  1545. else
  1546. em = btrfs_get_extent_fiemap(inode, NULL, 0,
  1547. start - root->sectorsize,
  1548. root->sectorsize, 0);
  1549. if (IS_ERR(em)) {
  1550. ret = -ENXIO;
  1551. goto out;
  1552. }
  1553. last_end = em->start + em->len;
  1554. if (em->block_start == EXTENT_MAP_DELALLOC)
  1555. last_end = min_t(u64, last_end, inode->i_size);
  1556. free_extent_map(em);
  1557. }
  1558. while (1) {
  1559. em = btrfs_get_extent_fiemap(inode, NULL, 0, start, len, 0);
  1560. if (IS_ERR(em)) {
  1561. ret = -ENXIO;
  1562. break;
  1563. }
  1564. if (em->block_start == EXTENT_MAP_HOLE) {
  1565. if (test_bit(EXTENT_FLAG_VACANCY, &em->flags)) {
  1566. if (last_end <= orig_start) {
  1567. free_extent_map(em);
  1568. ret = -ENXIO;
  1569. break;
  1570. }
  1571. }
  1572. if (origin == SEEK_HOLE) {
  1573. *offset = start;
  1574. free_extent_map(em);
  1575. break;
  1576. }
  1577. } else {
  1578. if (origin == SEEK_DATA) {
  1579. if (em->block_start == EXTENT_MAP_DELALLOC) {
  1580. if (start >= inode->i_size) {
  1581. free_extent_map(em);
  1582. ret = -ENXIO;
  1583. break;
  1584. }
  1585. }
  1586. *offset = start;
  1587. free_extent_map(em);
  1588. break;
  1589. }
  1590. }
  1591. start = em->start + em->len;
  1592. last_end = em->start + em->len;
  1593. if (em->block_start == EXTENT_MAP_DELALLOC)
  1594. last_end = min_t(u64, last_end, inode->i_size);
  1595. if (test_bit(EXTENT_FLAG_VACANCY, &em->flags)) {
  1596. free_extent_map(em);
  1597. ret = -ENXIO;
  1598. break;
  1599. }
  1600. free_extent_map(em);
  1601. cond_resched();
  1602. }
  1603. if (!ret)
  1604. *offset = min(*offset, inode->i_size);
  1605. out:
  1606. unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  1607. &cached_state, GFP_NOFS);
  1608. return ret;
  1609. }
  1610. static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int origin)
  1611. {
  1612. struct inode *inode = file->f_mapping->host;
  1613. int ret;
  1614. mutex_lock(&inode->i_mutex);
  1615. switch (origin) {
  1616. case SEEK_END:
  1617. case SEEK_CUR:
  1618. offset = generic_file_llseek(file, offset, origin);
  1619. goto out;
  1620. case SEEK_DATA:
  1621. case SEEK_HOLE:
  1622. if (offset >= i_size_read(inode)) {
  1623. mutex_unlock(&inode->i_mutex);
  1624. return -ENXIO;
  1625. }
  1626. ret = find_desired_extent(inode, &offset, origin);
  1627. if (ret) {
  1628. mutex_unlock(&inode->i_mutex);
  1629. return ret;
  1630. }
  1631. }
  1632. if (offset < 0 && !(file->f_mode & FMODE_UNSIGNED_OFFSET)) {
  1633. offset = -EINVAL;
  1634. goto out;
  1635. }
  1636. if (offset > inode->i_sb->s_maxbytes) {
  1637. offset = -EINVAL;
  1638. goto out;
  1639. }
  1640. /* Special lock needed here? */
  1641. if (offset != file->f_pos) {
  1642. file->f_pos = offset;
  1643. file->f_version = 0;
  1644. }
  1645. out:
  1646. mutex_unlock(&inode->i_mutex);
  1647. return offset;
  1648. }
  1649. const struct file_operations btrfs_file_operations = {
  1650. .llseek = btrfs_file_llseek,
  1651. .read = do_sync_read,
  1652. .write = do_sync_write,
  1653. .aio_read = generic_file_aio_read,
  1654. .splice_read = generic_file_splice_read,
  1655. .aio_write = btrfs_file_aio_write,
  1656. .mmap = btrfs_file_mmap,
  1657. .open = generic_file_open,
  1658. .release = btrfs_release_file,
  1659. .fsync = btrfs_sync_file,
  1660. .fallocate = btrfs_fallocate,
  1661. .unlocked_ioctl = btrfs_ioctl,
  1662. #ifdef CONFIG_COMPAT
  1663. .compat_ioctl = btrfs_ioctl,
  1664. #endif
  1665. };