file.c 43 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686
  1. /*
  2. * Copyright (C) 2007 Oracle. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public
  6. * License v2 as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. * General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public
  14. * License along with this program; if not, write to the
  15. * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16. * Boston, MA 021110-1307, USA.
  17. */
  18. #include <linux/fs.h>
  19. #include <linux/pagemap.h>
  20. #include <linux/highmem.h>
  21. #include <linux/time.h>
  22. #include <linux/init.h>
  23. #include <linux/string.h>
  24. #include <linux/backing-dev.h>
  25. #include <linux/mpage.h>
  26. #include <linux/falloc.h>
  27. #include <linux/swap.h>
  28. #include <linux/writeback.h>
  29. #include <linux/statfs.h>
  30. #include <linux/compat.h>
  31. #include <linux/slab.h>
  32. #include "ctree.h"
  33. #include "disk-io.h"
  34. #include "transaction.h"
  35. #include "btrfs_inode.h"
  36. #include "ioctl.h"
  37. #include "print-tree.h"
  38. #include "tree-log.h"
  39. #include "locking.h"
  40. #include "compat.h"
  41. /*
  42. * when auto defrag is enabled we
  43. * queue up these defrag structs to remember which
  44. * inodes need defragging passes
  45. */
  46. struct inode_defrag {
  47. struct rb_node rb_node;
  48. /* objectid */
  49. u64 ino;
  50. /*
  51. * transid where the defrag was added, we search for
  52. * extents newer than this
  53. */
  54. u64 transid;
  55. /* root objectid */
  56. u64 root;
  57. /* last offset we were able to defrag */
  58. u64 last_offset;
  59. /* if we've wrapped around back to zero once already */
  60. int cycled;
  61. };
  62. /* pop a record for an inode into the defrag tree. The lock
  63. * must be held already
  64. *
  65. * If you're inserting a record for an older transid than an
  66. * existing record, the transid already in the tree is lowered
  67. *
  68. * If an existing record is found the defrag item you
  69. * pass in is freed
  70. */
  71. static int __btrfs_add_inode_defrag(struct inode *inode,
  72. struct inode_defrag *defrag)
  73. {
  74. struct btrfs_root *root = BTRFS_I(inode)->root;
  75. struct inode_defrag *entry;
  76. struct rb_node **p;
  77. struct rb_node *parent = NULL;
  78. p = &root->fs_info->defrag_inodes.rb_node;
  79. while (*p) {
  80. parent = *p;
  81. entry = rb_entry(parent, struct inode_defrag, rb_node);
  82. if (defrag->ino < entry->ino)
  83. p = &parent->rb_left;
  84. else if (defrag->ino > entry->ino)
  85. p = &parent->rb_right;
  86. else {
  87. /* if we're reinserting an entry for
  88. * an old defrag run, make sure to
  89. * lower the transid of our existing record
  90. */
  91. if (defrag->transid < entry->transid)
  92. entry->transid = defrag->transid;
  93. if (defrag->last_offset > entry->last_offset)
  94. entry->last_offset = defrag->last_offset;
  95. goto exists;
  96. }
  97. }
  98. BTRFS_I(inode)->in_defrag = 1;
  99. rb_link_node(&defrag->rb_node, parent, p);
  100. rb_insert_color(&defrag->rb_node, &root->fs_info->defrag_inodes);
  101. return 0;
  102. exists:
  103. kfree(defrag);
  104. return 0;
  105. }
  106. /*
  107. * insert a defrag record for this inode if auto defrag is
  108. * enabled
  109. */
  110. int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
  111. struct inode *inode)
  112. {
  113. struct btrfs_root *root = BTRFS_I(inode)->root;
  114. struct inode_defrag *defrag;
  115. int ret = 0;
  116. u64 transid;
  117. if (!btrfs_test_opt(root, AUTO_DEFRAG))
  118. return 0;
  119. if (btrfs_fs_closing(root->fs_info))
  120. return 0;
  121. if (BTRFS_I(inode)->in_defrag)
  122. return 0;
  123. if (trans)
  124. transid = trans->transid;
  125. else
  126. transid = BTRFS_I(inode)->root->last_trans;
  127. defrag = kzalloc(sizeof(*defrag), GFP_NOFS);
  128. if (!defrag)
  129. return -ENOMEM;
  130. defrag->ino = btrfs_ino(inode);
  131. defrag->transid = transid;
  132. defrag->root = root->root_key.objectid;
  133. spin_lock(&root->fs_info->defrag_inodes_lock);
  134. if (!BTRFS_I(inode)->in_defrag)
  135. ret = __btrfs_add_inode_defrag(inode, defrag);
  136. spin_unlock(&root->fs_info->defrag_inodes_lock);
  137. return ret;
  138. }
  139. /*
  140. * must be called with the defrag_inodes lock held
  141. */
  142. struct inode_defrag *btrfs_find_defrag_inode(struct btrfs_fs_info *info, u64 ino,
  143. struct rb_node **next)
  144. {
  145. struct inode_defrag *entry = NULL;
  146. struct rb_node *p;
  147. struct rb_node *parent = NULL;
  148. p = info->defrag_inodes.rb_node;
  149. while (p) {
  150. parent = p;
  151. entry = rb_entry(parent, struct inode_defrag, rb_node);
  152. if (ino < entry->ino)
  153. p = parent->rb_left;
  154. else if (ino > entry->ino)
  155. p = parent->rb_right;
  156. else
  157. return entry;
  158. }
  159. if (next) {
  160. while (parent && ino > entry->ino) {
  161. parent = rb_next(parent);
  162. entry = rb_entry(parent, struct inode_defrag, rb_node);
  163. }
  164. *next = parent;
  165. }
  166. return NULL;
  167. }
  168. /*
  169. * run through the list of inodes in the FS that need
  170. * defragging
  171. */
  172. int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info)
  173. {
  174. struct inode_defrag *defrag;
  175. struct btrfs_root *inode_root;
  176. struct inode *inode;
  177. struct rb_node *n;
  178. struct btrfs_key key;
  179. struct btrfs_ioctl_defrag_range_args range;
  180. u64 first_ino = 0;
  181. int num_defrag;
  182. int defrag_batch = 1024;
  183. memset(&range, 0, sizeof(range));
  184. range.len = (u64)-1;
  185. atomic_inc(&fs_info->defrag_running);
  186. spin_lock(&fs_info->defrag_inodes_lock);
  187. while(1) {
  188. n = NULL;
  189. /* find an inode to defrag */
  190. defrag = btrfs_find_defrag_inode(fs_info, first_ino, &n);
  191. if (!defrag) {
  192. if (n)
  193. defrag = rb_entry(n, struct inode_defrag, rb_node);
  194. else if (first_ino) {
  195. first_ino = 0;
  196. continue;
  197. } else {
  198. break;
  199. }
  200. }
  201. /* remove it from the rbtree */
  202. first_ino = defrag->ino + 1;
  203. rb_erase(&defrag->rb_node, &fs_info->defrag_inodes);
  204. if (btrfs_fs_closing(fs_info))
  205. goto next_free;
  206. spin_unlock(&fs_info->defrag_inodes_lock);
  207. /* get the inode */
  208. key.objectid = defrag->root;
  209. btrfs_set_key_type(&key, BTRFS_ROOT_ITEM_KEY);
  210. key.offset = (u64)-1;
  211. inode_root = btrfs_read_fs_root_no_name(fs_info, &key);
  212. if (IS_ERR(inode_root))
  213. goto next;
  214. key.objectid = defrag->ino;
  215. btrfs_set_key_type(&key, BTRFS_INODE_ITEM_KEY);
  216. key.offset = 0;
  217. inode = btrfs_iget(fs_info->sb, &key, inode_root, NULL);
  218. if (IS_ERR(inode))
  219. goto next;
  220. /* do a chunk of defrag */
  221. BTRFS_I(inode)->in_defrag = 0;
  222. range.start = defrag->last_offset;
  223. num_defrag = btrfs_defrag_file(inode, NULL, &range, defrag->transid,
  224. defrag_batch);
  225. /*
  226. * if we filled the whole defrag batch, there
  227. * must be more work to do. Queue this defrag
  228. * again
  229. */
  230. if (num_defrag == defrag_batch) {
  231. defrag->last_offset = range.start;
  232. __btrfs_add_inode_defrag(inode, defrag);
  233. /*
  234. * we don't want to kfree defrag, we added it back to
  235. * the rbtree
  236. */
  237. defrag = NULL;
  238. } else if (defrag->last_offset && !defrag->cycled) {
  239. /*
  240. * we didn't fill our defrag batch, but
  241. * we didn't start at zero. Make sure we loop
  242. * around to the start of the file.
  243. */
  244. defrag->last_offset = 0;
  245. defrag->cycled = 1;
  246. __btrfs_add_inode_defrag(inode, defrag);
  247. defrag = NULL;
  248. }
  249. iput(inode);
  250. next:
  251. spin_lock(&fs_info->defrag_inodes_lock);
  252. next_free:
  253. kfree(defrag);
  254. }
  255. spin_unlock(&fs_info->defrag_inodes_lock);
  256. atomic_dec(&fs_info->defrag_running);
  257. /*
  258. * during unmount, we use the transaction_wait queue to
  259. * wait for the defragger to stop
  260. */
  261. wake_up(&fs_info->transaction_wait);
  262. return 0;
  263. }
  264. /* simple helper to fault in pages and copy. This should go away
  265. * and be replaced with calls into generic code.
  266. */
  267. static noinline int btrfs_copy_from_user(loff_t pos, int num_pages,
  268. size_t write_bytes,
  269. struct page **prepared_pages,
  270. struct iov_iter *i)
  271. {
  272. size_t copied = 0;
  273. size_t total_copied = 0;
  274. int pg = 0;
  275. int offset = pos & (PAGE_CACHE_SIZE - 1);
  276. while (write_bytes > 0) {
  277. size_t count = min_t(size_t,
  278. PAGE_CACHE_SIZE - offset, write_bytes);
  279. struct page *page = prepared_pages[pg];
  280. /*
  281. * Copy data from userspace to the current page
  282. *
  283. * Disable pagefault to avoid recursive lock since
  284. * the pages are already locked
  285. */
  286. pagefault_disable();
  287. copied = iov_iter_copy_from_user_atomic(page, i, offset, count);
  288. pagefault_enable();
  289. /* Flush processor's dcache for this page */
  290. flush_dcache_page(page);
  291. /*
  292. * if we get a partial write, we can end up with
  293. * partially up to date pages. These add
  294. * a lot of complexity, so make sure they don't
  295. * happen by forcing this copy to be retried.
  296. *
  297. * The rest of the btrfs_file_write code will fall
  298. * back to page at a time copies after we return 0.
  299. */
  300. if (!PageUptodate(page) && copied < count)
  301. copied = 0;
  302. iov_iter_advance(i, copied);
  303. write_bytes -= copied;
  304. total_copied += copied;
  305. /* Return to btrfs_file_aio_write to fault page */
  306. if (unlikely(copied == 0))
  307. break;
  308. if (unlikely(copied < PAGE_CACHE_SIZE - offset)) {
  309. offset += copied;
  310. } else {
  311. pg++;
  312. offset = 0;
  313. }
  314. }
  315. return total_copied;
  316. }
  317. /*
  318. * unlocks pages after btrfs_file_write is done with them
  319. */
  320. void btrfs_drop_pages(struct page **pages, size_t num_pages)
  321. {
  322. size_t i;
  323. for (i = 0; i < num_pages; i++) {
  324. /* page checked is some magic around finding pages that
  325. * have been modified without going through btrfs_set_page_dirty
  326. * clear it here
  327. */
  328. ClearPageChecked(pages[i]);
  329. unlock_page(pages[i]);
  330. mark_page_accessed(pages[i]);
  331. page_cache_release(pages[i]);
  332. }
  333. }
  334. /*
  335. * after copy_from_user, pages need to be dirtied and we need to make
  336. * sure holes are created between the current EOF and the start of
  337. * any next extents (if required).
  338. *
  339. * this also makes the decision about creating an inline extent vs
  340. * doing real data extents, marking pages dirty and delalloc as required.
  341. */
  342. int btrfs_dirty_pages(struct btrfs_root *root, struct inode *inode,
  343. struct page **pages, size_t num_pages,
  344. loff_t pos, size_t write_bytes,
  345. struct extent_state **cached)
  346. {
  347. int err = 0;
  348. int i;
  349. u64 num_bytes;
  350. u64 start_pos;
  351. u64 end_of_last_block;
  352. u64 end_pos = pos + write_bytes;
  353. loff_t isize = i_size_read(inode);
  354. start_pos = pos & ~((u64)root->sectorsize - 1);
  355. num_bytes = (write_bytes + pos - start_pos +
  356. root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
  357. end_of_last_block = start_pos + num_bytes - 1;
  358. err = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block,
  359. cached);
  360. if (err)
  361. return err;
  362. for (i = 0; i < num_pages; i++) {
  363. struct page *p = pages[i];
  364. SetPageUptodate(p);
  365. ClearPageChecked(p);
  366. set_page_dirty(p);
  367. }
  368. /*
  369. * we've only changed i_size in ram, and we haven't updated
  370. * the disk i_size. There is no need to log the inode
  371. * at this time.
  372. */
  373. if (end_pos > isize)
  374. i_size_write(inode, end_pos);
  375. return 0;
  376. }
  377. /*
  378. * this drops all the extents in the cache that intersect the range
  379. * [start, end]. Existing extents are split as required.
  380. */
  381. int btrfs_drop_extent_cache(struct inode *inode, u64 start, u64 end,
  382. int skip_pinned)
  383. {
  384. struct extent_map *em;
  385. struct extent_map *split = NULL;
  386. struct extent_map *split2 = NULL;
  387. struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
  388. u64 len = end - start + 1;
  389. int ret;
  390. int testend = 1;
  391. unsigned long flags;
  392. int compressed = 0;
  393. WARN_ON(end < start);
  394. if (end == (u64)-1) {
  395. len = (u64)-1;
  396. testend = 0;
  397. }
  398. while (1) {
  399. if (!split)
  400. split = alloc_extent_map();
  401. if (!split2)
  402. split2 = alloc_extent_map();
  403. BUG_ON(!split || !split2);
  404. write_lock(&em_tree->lock);
  405. em = lookup_extent_mapping(em_tree, start, len);
  406. if (!em) {
  407. write_unlock(&em_tree->lock);
  408. break;
  409. }
  410. flags = em->flags;
  411. if (skip_pinned && test_bit(EXTENT_FLAG_PINNED, &em->flags)) {
  412. if (testend && em->start + em->len >= start + len) {
  413. free_extent_map(em);
  414. write_unlock(&em_tree->lock);
  415. break;
  416. }
  417. start = em->start + em->len;
  418. if (testend)
  419. len = start + len - (em->start + em->len);
  420. free_extent_map(em);
  421. write_unlock(&em_tree->lock);
  422. continue;
  423. }
  424. compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
  425. clear_bit(EXTENT_FLAG_PINNED, &em->flags);
  426. remove_extent_mapping(em_tree, em);
  427. if (em->block_start < EXTENT_MAP_LAST_BYTE &&
  428. em->start < start) {
  429. split->start = em->start;
  430. split->len = start - em->start;
  431. split->orig_start = em->orig_start;
  432. split->block_start = em->block_start;
  433. if (compressed)
  434. split->block_len = em->block_len;
  435. else
  436. split->block_len = split->len;
  437. split->bdev = em->bdev;
  438. split->flags = flags;
  439. split->compress_type = em->compress_type;
  440. ret = add_extent_mapping(em_tree, split);
  441. BUG_ON(ret);
  442. free_extent_map(split);
  443. split = split2;
  444. split2 = NULL;
  445. }
  446. if (em->block_start < EXTENT_MAP_LAST_BYTE &&
  447. testend && em->start + em->len > start + len) {
  448. u64 diff = start + len - em->start;
  449. split->start = start + len;
  450. split->len = em->start + em->len - (start + len);
  451. split->bdev = em->bdev;
  452. split->flags = flags;
  453. split->compress_type = em->compress_type;
  454. if (compressed) {
  455. split->block_len = em->block_len;
  456. split->block_start = em->block_start;
  457. split->orig_start = em->orig_start;
  458. } else {
  459. split->block_len = split->len;
  460. split->block_start = em->block_start + diff;
  461. split->orig_start = split->start;
  462. }
  463. ret = add_extent_mapping(em_tree, split);
  464. BUG_ON(ret);
  465. free_extent_map(split);
  466. split = NULL;
  467. }
  468. write_unlock(&em_tree->lock);
  469. /* once for us */
  470. free_extent_map(em);
  471. /* once for the tree*/
  472. free_extent_map(em);
  473. }
  474. if (split)
  475. free_extent_map(split);
  476. if (split2)
  477. free_extent_map(split2);
  478. return 0;
  479. }
  480. /*
  481. * this is very complex, but the basic idea is to drop all extents
  482. * in the range start - end. hint_block is filled in with a block number
  483. * that would be a good hint to the block allocator for this file.
  484. *
  485. * If an extent intersects the range but is not entirely inside the range
  486. * it is either truncated or split. Anything entirely inside the range
  487. * is deleted from the tree.
  488. */
  489. int btrfs_drop_extents(struct btrfs_trans_handle *trans, struct inode *inode,
  490. u64 start, u64 end, u64 *hint_byte, int drop_cache)
  491. {
  492. struct btrfs_root *root = BTRFS_I(inode)->root;
  493. struct extent_buffer *leaf;
  494. struct btrfs_file_extent_item *fi;
  495. struct btrfs_path *path;
  496. struct btrfs_key key;
  497. struct btrfs_key new_key;
  498. u64 ino = btrfs_ino(inode);
  499. u64 search_start = start;
  500. u64 disk_bytenr = 0;
  501. u64 num_bytes = 0;
  502. u64 extent_offset = 0;
  503. u64 extent_end = 0;
  504. int del_nr = 0;
  505. int del_slot = 0;
  506. int extent_type;
  507. int recow;
  508. int ret;
  509. if (drop_cache)
  510. btrfs_drop_extent_cache(inode, start, end - 1, 0);
  511. path = btrfs_alloc_path();
  512. if (!path)
  513. return -ENOMEM;
  514. while (1) {
  515. recow = 0;
  516. ret = btrfs_lookup_file_extent(trans, root, path, ino,
  517. search_start, -1);
  518. if (ret < 0)
  519. break;
  520. if (ret > 0 && path->slots[0] > 0 && search_start == start) {
  521. leaf = path->nodes[0];
  522. btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
  523. if (key.objectid == ino &&
  524. key.type == BTRFS_EXTENT_DATA_KEY)
  525. path->slots[0]--;
  526. }
  527. ret = 0;
  528. next_slot:
  529. leaf = path->nodes[0];
  530. if (path->slots[0] >= btrfs_header_nritems(leaf)) {
  531. BUG_ON(del_nr > 0);
  532. ret = btrfs_next_leaf(root, path);
  533. if (ret < 0)
  534. break;
  535. if (ret > 0) {
  536. ret = 0;
  537. break;
  538. }
  539. leaf = path->nodes[0];
  540. recow = 1;
  541. }
  542. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  543. if (key.objectid > ino ||
  544. key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= end)
  545. break;
  546. fi = btrfs_item_ptr(leaf, path->slots[0],
  547. struct btrfs_file_extent_item);
  548. extent_type = btrfs_file_extent_type(leaf, fi);
  549. if (extent_type == BTRFS_FILE_EXTENT_REG ||
  550. extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
  551. disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  552. num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
  553. extent_offset = btrfs_file_extent_offset(leaf, fi);
  554. extent_end = key.offset +
  555. btrfs_file_extent_num_bytes(leaf, fi);
  556. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  557. extent_end = key.offset +
  558. btrfs_file_extent_inline_len(leaf, fi);
  559. } else {
  560. WARN_ON(1);
  561. extent_end = search_start;
  562. }
  563. if (extent_end <= search_start) {
  564. path->slots[0]++;
  565. goto next_slot;
  566. }
  567. search_start = max(key.offset, start);
  568. if (recow) {
  569. btrfs_release_path(path);
  570. continue;
  571. }
  572. /*
  573. * | - range to drop - |
  574. * | -------- extent -------- |
  575. */
  576. if (start > key.offset && end < extent_end) {
  577. BUG_ON(del_nr > 0);
  578. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  579. memcpy(&new_key, &key, sizeof(new_key));
  580. new_key.offset = start;
  581. ret = btrfs_duplicate_item(trans, root, path,
  582. &new_key);
  583. if (ret == -EAGAIN) {
  584. btrfs_release_path(path);
  585. continue;
  586. }
  587. if (ret < 0)
  588. break;
  589. leaf = path->nodes[0];
  590. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  591. struct btrfs_file_extent_item);
  592. btrfs_set_file_extent_num_bytes(leaf, fi,
  593. start - key.offset);
  594. fi = btrfs_item_ptr(leaf, path->slots[0],
  595. struct btrfs_file_extent_item);
  596. extent_offset += start - key.offset;
  597. btrfs_set_file_extent_offset(leaf, fi, extent_offset);
  598. btrfs_set_file_extent_num_bytes(leaf, fi,
  599. extent_end - start);
  600. btrfs_mark_buffer_dirty(leaf);
  601. if (disk_bytenr > 0) {
  602. ret = btrfs_inc_extent_ref(trans, root,
  603. disk_bytenr, num_bytes, 0,
  604. root->root_key.objectid,
  605. new_key.objectid,
  606. start - extent_offset);
  607. BUG_ON(ret);
  608. *hint_byte = disk_bytenr;
  609. }
  610. key.offset = start;
  611. }
  612. /*
  613. * | ---- range to drop ----- |
  614. * | -------- extent -------- |
  615. */
  616. if (start <= key.offset && end < extent_end) {
  617. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  618. memcpy(&new_key, &key, sizeof(new_key));
  619. new_key.offset = end;
  620. btrfs_set_item_key_safe(trans, root, path, &new_key);
  621. extent_offset += end - key.offset;
  622. btrfs_set_file_extent_offset(leaf, fi, extent_offset);
  623. btrfs_set_file_extent_num_bytes(leaf, fi,
  624. extent_end - end);
  625. btrfs_mark_buffer_dirty(leaf);
  626. if (disk_bytenr > 0) {
  627. inode_sub_bytes(inode, end - key.offset);
  628. *hint_byte = disk_bytenr;
  629. }
  630. break;
  631. }
  632. search_start = extent_end;
  633. /*
  634. * | ---- range to drop ----- |
  635. * | -------- extent -------- |
  636. */
  637. if (start > key.offset && end >= extent_end) {
  638. BUG_ON(del_nr > 0);
  639. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  640. btrfs_set_file_extent_num_bytes(leaf, fi,
  641. start - key.offset);
  642. btrfs_mark_buffer_dirty(leaf);
  643. if (disk_bytenr > 0) {
  644. inode_sub_bytes(inode, extent_end - start);
  645. *hint_byte = disk_bytenr;
  646. }
  647. if (end == extent_end)
  648. break;
  649. path->slots[0]++;
  650. goto next_slot;
  651. }
  652. /*
  653. * | ---- range to drop ----- |
  654. * | ------ extent ------ |
  655. */
  656. if (start <= key.offset && end >= extent_end) {
  657. if (del_nr == 0) {
  658. del_slot = path->slots[0];
  659. del_nr = 1;
  660. } else {
  661. BUG_ON(del_slot + del_nr != path->slots[0]);
  662. del_nr++;
  663. }
  664. if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  665. inode_sub_bytes(inode,
  666. extent_end - key.offset);
  667. extent_end = ALIGN(extent_end,
  668. root->sectorsize);
  669. } else if (disk_bytenr > 0) {
  670. ret = btrfs_free_extent(trans, root,
  671. disk_bytenr, num_bytes, 0,
  672. root->root_key.objectid,
  673. key.objectid, key.offset -
  674. extent_offset);
  675. BUG_ON(ret);
  676. inode_sub_bytes(inode,
  677. extent_end - key.offset);
  678. *hint_byte = disk_bytenr;
  679. }
  680. if (end == extent_end)
  681. break;
  682. if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {
  683. path->slots[0]++;
  684. goto next_slot;
  685. }
  686. ret = btrfs_del_items(trans, root, path, del_slot,
  687. del_nr);
  688. BUG_ON(ret);
  689. del_nr = 0;
  690. del_slot = 0;
  691. btrfs_release_path(path);
  692. continue;
  693. }
  694. BUG_ON(1);
  695. }
  696. if (del_nr > 0) {
  697. ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
  698. BUG_ON(ret);
  699. }
  700. btrfs_free_path(path);
  701. return ret;
  702. }
  703. static int extent_mergeable(struct extent_buffer *leaf, int slot,
  704. u64 objectid, u64 bytenr, u64 orig_offset,
  705. u64 *start, u64 *end)
  706. {
  707. struct btrfs_file_extent_item *fi;
  708. struct btrfs_key key;
  709. u64 extent_end;
  710. if (slot < 0 || slot >= btrfs_header_nritems(leaf))
  711. return 0;
  712. btrfs_item_key_to_cpu(leaf, &key, slot);
  713. if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)
  714. return 0;
  715. fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
  716. if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||
  717. btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||
  718. btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||
  719. btrfs_file_extent_compression(leaf, fi) ||
  720. btrfs_file_extent_encryption(leaf, fi) ||
  721. btrfs_file_extent_other_encoding(leaf, fi))
  722. return 0;
  723. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  724. if ((*start && *start != key.offset) || (*end && *end != extent_end))
  725. return 0;
  726. *start = key.offset;
  727. *end = extent_end;
  728. return 1;
  729. }
  730. /*
  731. * Mark extent in the range start - end as written.
  732. *
  733. * This changes extent type from 'pre-allocated' to 'regular'. If only
  734. * part of extent is marked as written, the extent will be split into
  735. * two or three.
  736. */
  737. int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
  738. struct inode *inode, u64 start, u64 end)
  739. {
  740. struct btrfs_root *root = BTRFS_I(inode)->root;
  741. struct extent_buffer *leaf;
  742. struct btrfs_path *path;
  743. struct btrfs_file_extent_item *fi;
  744. struct btrfs_key key;
  745. struct btrfs_key new_key;
  746. u64 bytenr;
  747. u64 num_bytes;
  748. u64 extent_end;
  749. u64 orig_offset;
  750. u64 other_start;
  751. u64 other_end;
  752. u64 split;
  753. int del_nr = 0;
  754. int del_slot = 0;
  755. int recow;
  756. int ret;
  757. u64 ino = btrfs_ino(inode);
  758. btrfs_drop_extent_cache(inode, start, end - 1, 0);
  759. path = btrfs_alloc_path();
  760. BUG_ON(!path);
  761. again:
  762. recow = 0;
  763. split = start;
  764. key.objectid = ino;
  765. key.type = BTRFS_EXTENT_DATA_KEY;
  766. key.offset = split;
  767. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  768. if (ret < 0)
  769. goto out;
  770. if (ret > 0 && path->slots[0] > 0)
  771. path->slots[0]--;
  772. leaf = path->nodes[0];
  773. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  774. BUG_ON(key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY);
  775. fi = btrfs_item_ptr(leaf, path->slots[0],
  776. struct btrfs_file_extent_item);
  777. BUG_ON(btrfs_file_extent_type(leaf, fi) !=
  778. BTRFS_FILE_EXTENT_PREALLOC);
  779. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  780. BUG_ON(key.offset > start || extent_end < end);
  781. bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  782. num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
  783. orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);
  784. memcpy(&new_key, &key, sizeof(new_key));
  785. if (start == key.offset && end < extent_end) {
  786. other_start = 0;
  787. other_end = start;
  788. if (extent_mergeable(leaf, path->slots[0] - 1,
  789. ino, bytenr, orig_offset,
  790. &other_start, &other_end)) {
  791. new_key.offset = end;
  792. btrfs_set_item_key_safe(trans, root, path, &new_key);
  793. fi = btrfs_item_ptr(leaf, path->slots[0],
  794. struct btrfs_file_extent_item);
  795. btrfs_set_file_extent_num_bytes(leaf, fi,
  796. extent_end - end);
  797. btrfs_set_file_extent_offset(leaf, fi,
  798. end - orig_offset);
  799. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  800. struct btrfs_file_extent_item);
  801. btrfs_set_file_extent_num_bytes(leaf, fi,
  802. end - other_start);
  803. btrfs_mark_buffer_dirty(leaf);
  804. goto out;
  805. }
  806. }
  807. if (start > key.offset && end == extent_end) {
  808. other_start = end;
  809. other_end = 0;
  810. if (extent_mergeable(leaf, path->slots[0] + 1,
  811. ino, bytenr, orig_offset,
  812. &other_start, &other_end)) {
  813. fi = btrfs_item_ptr(leaf, path->slots[0],
  814. struct btrfs_file_extent_item);
  815. btrfs_set_file_extent_num_bytes(leaf, fi,
  816. start - key.offset);
  817. path->slots[0]++;
  818. new_key.offset = start;
  819. btrfs_set_item_key_safe(trans, root, path, &new_key);
  820. fi = btrfs_item_ptr(leaf, path->slots[0],
  821. struct btrfs_file_extent_item);
  822. btrfs_set_file_extent_num_bytes(leaf, fi,
  823. other_end - start);
  824. btrfs_set_file_extent_offset(leaf, fi,
  825. start - orig_offset);
  826. btrfs_mark_buffer_dirty(leaf);
  827. goto out;
  828. }
  829. }
  830. while (start > key.offset || end < extent_end) {
  831. if (key.offset == start)
  832. split = end;
  833. new_key.offset = split;
  834. ret = btrfs_duplicate_item(trans, root, path, &new_key);
  835. if (ret == -EAGAIN) {
  836. btrfs_release_path(path);
  837. goto again;
  838. }
  839. BUG_ON(ret < 0);
  840. leaf = path->nodes[0];
  841. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  842. struct btrfs_file_extent_item);
  843. btrfs_set_file_extent_num_bytes(leaf, fi,
  844. split - key.offset);
  845. fi = btrfs_item_ptr(leaf, path->slots[0],
  846. struct btrfs_file_extent_item);
  847. btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);
  848. btrfs_set_file_extent_num_bytes(leaf, fi,
  849. extent_end - split);
  850. btrfs_mark_buffer_dirty(leaf);
  851. ret = btrfs_inc_extent_ref(trans, root, bytenr, num_bytes, 0,
  852. root->root_key.objectid,
  853. ino, orig_offset);
  854. BUG_ON(ret);
  855. if (split == start) {
  856. key.offset = start;
  857. } else {
  858. BUG_ON(start != key.offset);
  859. path->slots[0]--;
  860. extent_end = end;
  861. }
  862. recow = 1;
  863. }
  864. other_start = end;
  865. other_end = 0;
  866. if (extent_mergeable(leaf, path->slots[0] + 1,
  867. ino, bytenr, orig_offset,
  868. &other_start, &other_end)) {
  869. if (recow) {
  870. btrfs_release_path(path);
  871. goto again;
  872. }
  873. extent_end = other_end;
  874. del_slot = path->slots[0] + 1;
  875. del_nr++;
  876. ret = btrfs_free_extent(trans, root, bytenr, num_bytes,
  877. 0, root->root_key.objectid,
  878. ino, orig_offset);
  879. BUG_ON(ret);
  880. }
  881. other_start = 0;
  882. other_end = start;
  883. if (extent_mergeable(leaf, path->slots[0] - 1,
  884. ino, bytenr, orig_offset,
  885. &other_start, &other_end)) {
  886. if (recow) {
  887. btrfs_release_path(path);
  888. goto again;
  889. }
  890. key.offset = other_start;
  891. del_slot = path->slots[0];
  892. del_nr++;
  893. ret = btrfs_free_extent(trans, root, bytenr, num_bytes,
  894. 0, root->root_key.objectid,
  895. ino, orig_offset);
  896. BUG_ON(ret);
  897. }
  898. if (del_nr == 0) {
  899. fi = btrfs_item_ptr(leaf, path->slots[0],
  900. struct btrfs_file_extent_item);
  901. btrfs_set_file_extent_type(leaf, fi,
  902. BTRFS_FILE_EXTENT_REG);
  903. btrfs_mark_buffer_dirty(leaf);
  904. } else {
  905. fi = btrfs_item_ptr(leaf, del_slot - 1,
  906. struct btrfs_file_extent_item);
  907. btrfs_set_file_extent_type(leaf, fi,
  908. BTRFS_FILE_EXTENT_REG);
  909. btrfs_set_file_extent_num_bytes(leaf, fi,
  910. extent_end - key.offset);
  911. btrfs_mark_buffer_dirty(leaf);
  912. ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
  913. BUG_ON(ret);
  914. }
  915. out:
  916. btrfs_free_path(path);
  917. return 0;
  918. }
  919. /*
  920. * on error we return an unlocked page and the error value
  921. * on success we return a locked page and 0
  922. */
  923. static int prepare_uptodate_page(struct page *page, u64 pos)
  924. {
  925. int ret = 0;
  926. if ((pos & (PAGE_CACHE_SIZE - 1)) && !PageUptodate(page)) {
  927. ret = btrfs_readpage(NULL, page);
  928. if (ret)
  929. return ret;
  930. lock_page(page);
  931. if (!PageUptodate(page)) {
  932. unlock_page(page);
  933. return -EIO;
  934. }
  935. }
  936. return 0;
  937. }
  938. /*
  939. * this gets pages into the page cache and locks them down, it also properly
  940. * waits for data=ordered extents to finish before allowing the pages to be
  941. * modified.
  942. */
  943. static noinline int prepare_pages(struct btrfs_root *root, struct file *file,
  944. struct page **pages, size_t num_pages,
  945. loff_t pos, unsigned long first_index,
  946. unsigned long last_index, size_t write_bytes)
  947. {
  948. struct extent_state *cached_state = NULL;
  949. int i;
  950. unsigned long index = pos >> PAGE_CACHE_SHIFT;
  951. struct inode *inode = fdentry(file)->d_inode;
  952. int err = 0;
  953. int faili = 0;
  954. u64 start_pos;
  955. u64 last_pos;
  956. start_pos = pos & ~((u64)root->sectorsize - 1);
  957. last_pos = ((u64)index + num_pages) << PAGE_CACHE_SHIFT;
  958. if (start_pos > inode->i_size) {
  959. err = btrfs_cont_expand(inode, i_size_read(inode), start_pos);
  960. if (err)
  961. return err;
  962. }
  963. again:
  964. for (i = 0; i < num_pages; i++) {
  965. pages[i] = find_or_create_page(inode->i_mapping, index + i,
  966. GFP_NOFS);
  967. if (!pages[i]) {
  968. faili = i - 1;
  969. err = -ENOMEM;
  970. goto fail;
  971. }
  972. if (i == 0)
  973. err = prepare_uptodate_page(pages[i], pos);
  974. if (i == num_pages - 1)
  975. err = prepare_uptodate_page(pages[i],
  976. pos + write_bytes);
  977. if (err) {
  978. page_cache_release(pages[i]);
  979. faili = i - 1;
  980. goto fail;
  981. }
  982. wait_on_page_writeback(pages[i]);
  983. }
  984. err = 0;
  985. if (start_pos < inode->i_size) {
  986. struct btrfs_ordered_extent *ordered;
  987. lock_extent_bits(&BTRFS_I(inode)->io_tree,
  988. start_pos, last_pos - 1, 0, &cached_state,
  989. GFP_NOFS);
  990. ordered = btrfs_lookup_first_ordered_extent(inode,
  991. last_pos - 1);
  992. if (ordered &&
  993. ordered->file_offset + ordered->len > start_pos &&
  994. ordered->file_offset < last_pos) {
  995. btrfs_put_ordered_extent(ordered);
  996. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  997. start_pos, last_pos - 1,
  998. &cached_state, GFP_NOFS);
  999. for (i = 0; i < num_pages; i++) {
  1000. unlock_page(pages[i]);
  1001. page_cache_release(pages[i]);
  1002. }
  1003. btrfs_wait_ordered_range(inode, start_pos,
  1004. last_pos - start_pos);
  1005. goto again;
  1006. }
  1007. if (ordered)
  1008. btrfs_put_ordered_extent(ordered);
  1009. clear_extent_bit(&BTRFS_I(inode)->io_tree, start_pos,
  1010. last_pos - 1, EXTENT_DIRTY | EXTENT_DELALLOC |
  1011. EXTENT_DO_ACCOUNTING, 0, 0, &cached_state,
  1012. GFP_NOFS);
  1013. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  1014. start_pos, last_pos - 1, &cached_state,
  1015. GFP_NOFS);
  1016. }
  1017. for (i = 0; i < num_pages; i++) {
  1018. clear_page_dirty_for_io(pages[i]);
  1019. set_page_extent_mapped(pages[i]);
  1020. WARN_ON(!PageLocked(pages[i]));
  1021. }
  1022. return 0;
  1023. fail:
  1024. while (faili >= 0) {
  1025. unlock_page(pages[faili]);
  1026. page_cache_release(pages[faili]);
  1027. faili--;
  1028. }
  1029. return err;
  1030. }
  1031. static noinline ssize_t __btrfs_buffered_write(struct file *file,
  1032. struct iov_iter *i,
  1033. loff_t pos)
  1034. {
  1035. struct inode *inode = fdentry(file)->d_inode;
  1036. struct btrfs_root *root = BTRFS_I(inode)->root;
  1037. struct page **pages = NULL;
  1038. unsigned long first_index;
  1039. unsigned long last_index;
  1040. size_t num_written = 0;
  1041. int nrptrs;
  1042. int ret = 0;
  1043. nrptrs = min((iov_iter_count(i) + PAGE_CACHE_SIZE - 1) /
  1044. PAGE_CACHE_SIZE, PAGE_CACHE_SIZE /
  1045. (sizeof(struct page *)));
  1046. pages = kmalloc(nrptrs * sizeof(struct page *), GFP_KERNEL);
  1047. if (!pages)
  1048. return -ENOMEM;
  1049. first_index = pos >> PAGE_CACHE_SHIFT;
  1050. last_index = (pos + iov_iter_count(i)) >> PAGE_CACHE_SHIFT;
  1051. while (iov_iter_count(i) > 0) {
  1052. size_t offset = pos & (PAGE_CACHE_SIZE - 1);
  1053. size_t write_bytes = min(iov_iter_count(i),
  1054. nrptrs * (size_t)PAGE_CACHE_SIZE -
  1055. offset);
  1056. size_t num_pages = (write_bytes + offset +
  1057. PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1058. size_t dirty_pages;
  1059. size_t copied;
  1060. WARN_ON(num_pages > nrptrs);
  1061. /*
  1062. * Fault pages before locking them in prepare_pages
  1063. * to avoid recursive lock
  1064. */
  1065. if (unlikely(iov_iter_fault_in_readable(i, write_bytes))) {
  1066. ret = -EFAULT;
  1067. break;
  1068. }
  1069. ret = btrfs_delalloc_reserve_space(inode,
  1070. num_pages << PAGE_CACHE_SHIFT);
  1071. if (ret)
  1072. break;
  1073. /*
  1074. * This is going to setup the pages array with the number of
  1075. * pages we want, so we don't really need to worry about the
  1076. * contents of pages from loop to loop
  1077. */
  1078. ret = prepare_pages(root, file, pages, num_pages,
  1079. pos, first_index, last_index,
  1080. write_bytes);
  1081. if (ret) {
  1082. btrfs_delalloc_release_space(inode,
  1083. num_pages << PAGE_CACHE_SHIFT);
  1084. break;
  1085. }
  1086. copied = btrfs_copy_from_user(pos, num_pages,
  1087. write_bytes, pages, i);
  1088. /*
  1089. * if we have trouble faulting in the pages, fall
  1090. * back to one page at a time
  1091. */
  1092. if (copied < write_bytes)
  1093. nrptrs = 1;
  1094. if (copied == 0)
  1095. dirty_pages = 0;
  1096. else
  1097. dirty_pages = (copied + offset +
  1098. PAGE_CACHE_SIZE - 1) >>
  1099. PAGE_CACHE_SHIFT;
  1100. /*
  1101. * If we had a short copy we need to release the excess delaloc
  1102. * bytes we reserved. We need to increment outstanding_extents
  1103. * because btrfs_delalloc_release_space will decrement it, but
  1104. * we still have an outstanding extent for the chunk we actually
  1105. * managed to copy.
  1106. */
  1107. if (num_pages > dirty_pages) {
  1108. if (copied > 0) {
  1109. spin_lock(&BTRFS_I(inode)->lock);
  1110. BTRFS_I(inode)->outstanding_extents++;
  1111. spin_unlock(&BTRFS_I(inode)->lock);
  1112. }
  1113. btrfs_delalloc_release_space(inode,
  1114. (num_pages - dirty_pages) <<
  1115. PAGE_CACHE_SHIFT);
  1116. }
  1117. if (copied > 0) {
  1118. ret = btrfs_dirty_pages(root, inode, pages,
  1119. dirty_pages, pos, copied,
  1120. NULL);
  1121. if (ret) {
  1122. btrfs_delalloc_release_space(inode,
  1123. dirty_pages << PAGE_CACHE_SHIFT);
  1124. btrfs_drop_pages(pages, num_pages);
  1125. break;
  1126. }
  1127. }
  1128. btrfs_drop_pages(pages, num_pages);
  1129. cond_resched();
  1130. balance_dirty_pages_ratelimited_nr(inode->i_mapping,
  1131. dirty_pages);
  1132. if (dirty_pages < (root->leafsize >> PAGE_CACHE_SHIFT) + 1)
  1133. btrfs_btree_balance_dirty(root, 1);
  1134. btrfs_throttle(root);
  1135. pos += copied;
  1136. num_written += copied;
  1137. }
  1138. kfree(pages);
  1139. return num_written ? num_written : ret;
  1140. }
  1141. static ssize_t __btrfs_direct_write(struct kiocb *iocb,
  1142. const struct iovec *iov,
  1143. unsigned long nr_segs, loff_t pos,
  1144. loff_t *ppos, size_t count, size_t ocount)
  1145. {
  1146. struct file *file = iocb->ki_filp;
  1147. struct inode *inode = fdentry(file)->d_inode;
  1148. struct iov_iter i;
  1149. ssize_t written;
  1150. ssize_t written_buffered;
  1151. loff_t endbyte;
  1152. int err;
  1153. written = generic_file_direct_write(iocb, iov, &nr_segs, pos, ppos,
  1154. count, ocount);
  1155. /*
  1156. * the generic O_DIRECT will update in-memory i_size after the
  1157. * DIOs are done. But our endio handlers that update the on
  1158. * disk i_size never update past the in memory i_size. So we
  1159. * need one more update here to catch any additions to the
  1160. * file
  1161. */
  1162. if (inode->i_size != BTRFS_I(inode)->disk_i_size) {
  1163. btrfs_ordered_update_i_size(inode, inode->i_size, NULL);
  1164. mark_inode_dirty(inode);
  1165. }
  1166. if (written < 0 || written == count)
  1167. return written;
  1168. pos += written;
  1169. count -= written;
  1170. iov_iter_init(&i, iov, nr_segs, count, written);
  1171. written_buffered = __btrfs_buffered_write(file, &i, pos);
  1172. if (written_buffered < 0) {
  1173. err = written_buffered;
  1174. goto out;
  1175. }
  1176. endbyte = pos + written_buffered - 1;
  1177. err = filemap_write_and_wait_range(file->f_mapping, pos, endbyte);
  1178. if (err)
  1179. goto out;
  1180. written += written_buffered;
  1181. *ppos = pos + written_buffered;
  1182. invalidate_mapping_pages(file->f_mapping, pos >> PAGE_CACHE_SHIFT,
  1183. endbyte >> PAGE_CACHE_SHIFT);
  1184. out:
  1185. return written ? written : err;
  1186. }
  1187. static ssize_t btrfs_file_aio_write(struct kiocb *iocb,
  1188. const struct iovec *iov,
  1189. unsigned long nr_segs, loff_t pos)
  1190. {
  1191. struct file *file = iocb->ki_filp;
  1192. struct inode *inode = fdentry(file)->d_inode;
  1193. struct btrfs_root *root = BTRFS_I(inode)->root;
  1194. loff_t *ppos = &iocb->ki_pos;
  1195. ssize_t num_written = 0;
  1196. ssize_t err = 0;
  1197. size_t count, ocount;
  1198. vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE);
  1199. mutex_lock(&inode->i_mutex);
  1200. err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ);
  1201. if (err) {
  1202. mutex_unlock(&inode->i_mutex);
  1203. goto out;
  1204. }
  1205. count = ocount;
  1206. current->backing_dev_info = inode->i_mapping->backing_dev_info;
  1207. err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
  1208. if (err) {
  1209. mutex_unlock(&inode->i_mutex);
  1210. goto out;
  1211. }
  1212. if (count == 0) {
  1213. mutex_unlock(&inode->i_mutex);
  1214. goto out;
  1215. }
  1216. err = file_remove_suid(file);
  1217. if (err) {
  1218. mutex_unlock(&inode->i_mutex);
  1219. goto out;
  1220. }
  1221. /*
  1222. * If BTRFS flips readonly due to some impossible error
  1223. * (fs_info->fs_state now has BTRFS_SUPER_FLAG_ERROR),
  1224. * although we have opened a file as writable, we have
  1225. * to stop this write operation to ensure FS consistency.
  1226. */
  1227. if (root->fs_info->fs_state & BTRFS_SUPER_FLAG_ERROR) {
  1228. mutex_unlock(&inode->i_mutex);
  1229. err = -EROFS;
  1230. goto out;
  1231. }
  1232. file_update_time(file);
  1233. BTRFS_I(inode)->sequence++;
  1234. if (unlikely(file->f_flags & O_DIRECT)) {
  1235. num_written = __btrfs_direct_write(iocb, iov, nr_segs,
  1236. pos, ppos, count, ocount);
  1237. } else {
  1238. struct iov_iter i;
  1239. iov_iter_init(&i, iov, nr_segs, count, num_written);
  1240. num_written = __btrfs_buffered_write(file, &i, pos);
  1241. if (num_written > 0)
  1242. *ppos = pos + num_written;
  1243. }
  1244. mutex_unlock(&inode->i_mutex);
  1245. /*
  1246. * we want to make sure fsync finds this change
  1247. * but we haven't joined a transaction running right now.
  1248. *
  1249. * Later on, someone is sure to update the inode and get the
  1250. * real transid recorded.
  1251. *
  1252. * We set last_trans now to the fs_info generation + 1,
  1253. * this will either be one more than the running transaction
  1254. * or the generation used for the next transaction if there isn't
  1255. * one running right now.
  1256. */
  1257. BTRFS_I(inode)->last_trans = root->fs_info->generation + 1;
  1258. if (num_written > 0 || num_written == -EIOCBQUEUED) {
  1259. err = generic_write_sync(file, pos, num_written);
  1260. if (err < 0 && num_written > 0)
  1261. num_written = err;
  1262. }
  1263. out:
  1264. current->backing_dev_info = NULL;
  1265. return num_written ? num_written : err;
  1266. }
  1267. int btrfs_release_file(struct inode *inode, struct file *filp)
  1268. {
  1269. /*
  1270. * ordered_data_close is set by settattr when we are about to truncate
  1271. * a file from a non-zero size to a zero size. This tries to
  1272. * flush down new bytes that may have been written if the
  1273. * application were using truncate to replace a file in place.
  1274. */
  1275. if (BTRFS_I(inode)->ordered_data_close) {
  1276. BTRFS_I(inode)->ordered_data_close = 0;
  1277. btrfs_add_ordered_operation(NULL, BTRFS_I(inode)->root, inode);
  1278. if (inode->i_size > BTRFS_ORDERED_OPERATIONS_FLUSH_LIMIT)
  1279. filemap_flush(inode->i_mapping);
  1280. }
  1281. if (filp->private_data)
  1282. btrfs_ioctl_trans_end(filp);
  1283. return 0;
  1284. }
  1285. /*
  1286. * fsync call for both files and directories. This logs the inode into
  1287. * the tree log instead of forcing full commits whenever possible.
  1288. *
  1289. * It needs to call filemap_fdatawait so that all ordered extent updates are
  1290. * in the metadata btree are up to date for copying to the log.
  1291. *
  1292. * It drops the inode mutex before doing the tree log commit. This is an
  1293. * important optimization for directories because holding the mutex prevents
  1294. * new operations on the dir while we write to disk.
  1295. */
  1296. int btrfs_sync_file(struct file *file, int datasync)
  1297. {
  1298. struct dentry *dentry = file->f_path.dentry;
  1299. struct inode *inode = dentry->d_inode;
  1300. struct btrfs_root *root = BTRFS_I(inode)->root;
  1301. int ret = 0;
  1302. struct btrfs_trans_handle *trans;
  1303. trace_btrfs_sync_file(file, datasync);
  1304. /* we wait first, since the writeback may change the inode */
  1305. root->log_batch++;
  1306. /* the VFS called filemap_fdatawrite for us */
  1307. btrfs_wait_ordered_range(inode, 0, (u64)-1);
  1308. root->log_batch++;
  1309. /*
  1310. * check the transaction that last modified this inode
  1311. * and see if its already been committed
  1312. */
  1313. if (!BTRFS_I(inode)->last_trans)
  1314. goto out;
  1315. /*
  1316. * if the last transaction that changed this file was before
  1317. * the current transaction, we can bail out now without any
  1318. * syncing
  1319. */
  1320. smp_mb();
  1321. if (BTRFS_I(inode)->last_trans <=
  1322. root->fs_info->last_trans_committed) {
  1323. BTRFS_I(inode)->last_trans = 0;
  1324. goto out;
  1325. }
  1326. /*
  1327. * ok we haven't committed the transaction yet, lets do a commit
  1328. */
  1329. if (file->private_data)
  1330. btrfs_ioctl_trans_end(file);
  1331. trans = btrfs_start_transaction(root, 0);
  1332. if (IS_ERR(trans)) {
  1333. ret = PTR_ERR(trans);
  1334. goto out;
  1335. }
  1336. ret = btrfs_log_dentry_safe(trans, root, dentry);
  1337. if (ret < 0)
  1338. goto out;
  1339. /* we've logged all the items and now have a consistent
  1340. * version of the file in the log. It is possible that
  1341. * someone will come in and modify the file, but that's
  1342. * fine because the log is consistent on disk, and we
  1343. * have references to all of the file's extents
  1344. *
  1345. * It is possible that someone will come in and log the
  1346. * file again, but that will end up using the synchronization
  1347. * inside btrfs_sync_log to keep things safe.
  1348. */
  1349. mutex_unlock(&dentry->d_inode->i_mutex);
  1350. if (ret != BTRFS_NO_LOG_SYNC) {
  1351. if (ret > 0) {
  1352. ret = btrfs_commit_transaction(trans, root);
  1353. } else {
  1354. ret = btrfs_sync_log(trans, root);
  1355. if (ret == 0)
  1356. ret = btrfs_end_transaction(trans, root);
  1357. else
  1358. ret = btrfs_commit_transaction(trans, root);
  1359. }
  1360. } else {
  1361. ret = btrfs_end_transaction(trans, root);
  1362. }
  1363. mutex_lock(&dentry->d_inode->i_mutex);
  1364. out:
  1365. return ret > 0 ? -EIO : ret;
  1366. }
  1367. static const struct vm_operations_struct btrfs_file_vm_ops = {
  1368. .fault = filemap_fault,
  1369. .page_mkwrite = btrfs_page_mkwrite,
  1370. };
  1371. static int btrfs_file_mmap(struct file *filp, struct vm_area_struct *vma)
  1372. {
  1373. struct address_space *mapping = filp->f_mapping;
  1374. if (!mapping->a_ops->readpage)
  1375. return -ENOEXEC;
  1376. file_accessed(filp);
  1377. vma->vm_ops = &btrfs_file_vm_ops;
  1378. vma->vm_flags |= VM_CAN_NONLINEAR;
  1379. return 0;
  1380. }
  1381. static long btrfs_fallocate(struct file *file, int mode,
  1382. loff_t offset, loff_t len)
  1383. {
  1384. struct inode *inode = file->f_path.dentry->d_inode;
  1385. struct extent_state *cached_state = NULL;
  1386. u64 cur_offset;
  1387. u64 last_byte;
  1388. u64 alloc_start;
  1389. u64 alloc_end;
  1390. u64 alloc_hint = 0;
  1391. u64 locked_end;
  1392. u64 mask = BTRFS_I(inode)->root->sectorsize - 1;
  1393. struct extent_map *em;
  1394. int ret;
  1395. alloc_start = offset & ~mask;
  1396. alloc_end = (offset + len + mask) & ~mask;
  1397. /* We only support the FALLOC_FL_KEEP_SIZE mode */
  1398. if (mode & ~FALLOC_FL_KEEP_SIZE)
  1399. return -EOPNOTSUPP;
  1400. /*
  1401. * wait for ordered IO before we have any locks. We'll loop again
  1402. * below with the locks held.
  1403. */
  1404. btrfs_wait_ordered_range(inode, alloc_start, alloc_end - alloc_start);
  1405. mutex_lock(&inode->i_mutex);
  1406. ret = inode_newsize_ok(inode, alloc_end);
  1407. if (ret)
  1408. goto out;
  1409. if (alloc_start > inode->i_size) {
  1410. ret = btrfs_cont_expand(inode, i_size_read(inode),
  1411. alloc_start);
  1412. if (ret)
  1413. goto out;
  1414. }
  1415. ret = btrfs_check_data_free_space(inode, alloc_end - alloc_start);
  1416. if (ret)
  1417. goto out;
  1418. locked_end = alloc_end - 1;
  1419. while (1) {
  1420. struct btrfs_ordered_extent *ordered;
  1421. /* the extent lock is ordered inside the running
  1422. * transaction
  1423. */
  1424. lock_extent_bits(&BTRFS_I(inode)->io_tree, alloc_start,
  1425. locked_end, 0, &cached_state, GFP_NOFS);
  1426. ordered = btrfs_lookup_first_ordered_extent(inode,
  1427. alloc_end - 1);
  1428. if (ordered &&
  1429. ordered->file_offset + ordered->len > alloc_start &&
  1430. ordered->file_offset < alloc_end) {
  1431. btrfs_put_ordered_extent(ordered);
  1432. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  1433. alloc_start, locked_end,
  1434. &cached_state, GFP_NOFS);
  1435. /*
  1436. * we can't wait on the range with the transaction
  1437. * running or with the extent lock held
  1438. */
  1439. btrfs_wait_ordered_range(inode, alloc_start,
  1440. alloc_end - alloc_start);
  1441. } else {
  1442. if (ordered)
  1443. btrfs_put_ordered_extent(ordered);
  1444. break;
  1445. }
  1446. }
  1447. cur_offset = alloc_start;
  1448. while (1) {
  1449. em = btrfs_get_extent(inode, NULL, 0, cur_offset,
  1450. alloc_end - cur_offset, 0);
  1451. BUG_ON(IS_ERR_OR_NULL(em));
  1452. last_byte = min(extent_map_end(em), alloc_end);
  1453. last_byte = (last_byte + mask) & ~mask;
  1454. if (em->block_start == EXTENT_MAP_HOLE ||
  1455. (cur_offset >= inode->i_size &&
  1456. !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
  1457. ret = btrfs_prealloc_file_range(inode, mode, cur_offset,
  1458. last_byte - cur_offset,
  1459. 1 << inode->i_blkbits,
  1460. offset + len,
  1461. &alloc_hint);
  1462. if (ret < 0) {
  1463. free_extent_map(em);
  1464. break;
  1465. }
  1466. }
  1467. free_extent_map(em);
  1468. cur_offset = last_byte;
  1469. if (cur_offset >= alloc_end) {
  1470. ret = 0;
  1471. break;
  1472. }
  1473. }
  1474. unlock_extent_cached(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
  1475. &cached_state, GFP_NOFS);
  1476. btrfs_free_reserved_data_space(inode, alloc_end - alloc_start);
  1477. out:
  1478. mutex_unlock(&inode->i_mutex);
  1479. return ret;
  1480. }
  1481. const struct file_operations btrfs_file_operations = {
  1482. .llseek = generic_file_llseek,
  1483. .read = do_sync_read,
  1484. .write = do_sync_write,
  1485. .aio_read = generic_file_aio_read,
  1486. .splice_read = generic_file_splice_read,
  1487. .aio_write = btrfs_file_aio_write,
  1488. .mmap = btrfs_file_mmap,
  1489. .open = generic_file_open,
  1490. .release = btrfs_release_file,
  1491. .fsync = btrfs_sync_file,
  1492. .fallocate = btrfs_fallocate,
  1493. .unlocked_ioctl = btrfs_ioctl,
  1494. #ifdef CONFIG_COMPAT
  1495. .compat_ioctl = btrfs_ioctl,
  1496. #endif
  1497. };