aio.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418
  1. /*
  2. * Copyright (C) 2004 Jeff Dike (jdike@addtoit.com)
  3. * Licensed under the GPL
  4. */
  5. #include <stdlib.h>
  6. #include <unistd.h>
  7. #include <signal.h>
  8. #include <string.h>
  9. #include <errno.h>
  10. #include <sched.h>
  11. #include <sys/syscall.h>
  12. #include "os.h"
  13. #include "helper.h"
  14. #include "aio.h"
  15. #include "init.h"
  16. #include "user.h"
  17. #include "mode.h"
  18. static int aio_req_fd_r = -1;
  19. static int aio_req_fd_w = -1;
  20. static int update_aio(struct aio_context *aio, int res)
  21. {
  22. if(res < 0)
  23. aio->len = res;
  24. else if((res == 0) && (aio->type == AIO_READ)){
  25. /* This is the EOF case - we have hit the end of the file
  26. * and it ends in a partial block, so we fill the end of
  27. * the block with zeros and claim success.
  28. */
  29. memset(aio->data, 0, aio->len);
  30. aio->len = 0;
  31. }
  32. else if(res > 0){
  33. aio->len -= res;
  34. aio->data += res;
  35. aio->offset += res;
  36. return aio->len;
  37. }
  38. return 0;
  39. }
  40. #if defined(HAVE_AIO_ABI)
  41. #include <linux/aio_abi.h>
  42. /* If we have the headers, we are going to build with AIO enabled.
  43. * If we don't have aio in libc, we define the necessary stubs here.
  44. */
  45. #if !defined(HAVE_AIO_LIBC)
  46. static long io_setup(int n, aio_context_t *ctxp)
  47. {
  48. return syscall(__NR_io_setup, n, ctxp);
  49. }
  50. static long io_submit(aio_context_t ctx, long nr, struct iocb **iocbpp)
  51. {
  52. return syscall(__NR_io_submit, ctx, nr, iocbpp);
  53. }
  54. static long io_getevents(aio_context_t ctx_id, long min_nr, long nr,
  55. struct io_event *events, struct timespec *timeout)
  56. {
  57. return syscall(__NR_io_getevents, ctx_id, min_nr, nr, events, timeout);
  58. }
  59. #endif
  60. /* The AIO_MMAP cases force the mmapped page into memory here
  61. * rather than in whatever place first touches the data. I used
  62. * to do this by touching the page, but that's delicate because
  63. * gcc is prone to optimizing that away. So, what's done here
  64. * is we read from the descriptor from which the page was
  65. * mapped. The caller is required to pass an offset which is
  66. * inside the page that was mapped. Thus, when the read
  67. * returns, we know that the page is in the page cache, and
  68. * that it now backs the mmapped area.
  69. */
  70. static int do_aio(aio_context_t ctx, struct aio_context *aio)
  71. {
  72. struct iocb iocb, *iocbp = &iocb;
  73. char c;
  74. int err;
  75. iocb = ((struct iocb) { .aio_data = (unsigned long) aio,
  76. .aio_reqprio = 0,
  77. .aio_fildes = aio->fd,
  78. .aio_buf = (unsigned long) aio->data,
  79. .aio_nbytes = aio->len,
  80. .aio_offset = aio->offset,
  81. .aio_reserved1 = 0,
  82. .aio_reserved2 = 0,
  83. .aio_reserved3 = 0 });
  84. switch(aio->type){
  85. case AIO_READ:
  86. iocb.aio_lio_opcode = IOCB_CMD_PREAD;
  87. break;
  88. case AIO_WRITE:
  89. iocb.aio_lio_opcode = IOCB_CMD_PWRITE;
  90. break;
  91. case AIO_MMAP:
  92. iocb.aio_lio_opcode = IOCB_CMD_PREAD;
  93. iocb.aio_buf = (unsigned long) &c;
  94. iocb.aio_nbytes = sizeof(c);
  95. break;
  96. default:
  97. printk("Bogus op in do_aio - %d\n", aio->type);
  98. err = -EINVAL;
  99. goto out;
  100. }
  101. err = io_submit(ctx, 1, &iocbp);
  102. if(err > 0)
  103. err = 0;
  104. else
  105. err = -errno;
  106. out:
  107. return err;
  108. }
  109. static aio_context_t ctx = 0;
  110. static int aio_thread(void *arg)
  111. {
  112. struct aio_thread_reply reply;
  113. struct aio_context *aio;
  114. struct io_event event;
  115. int err, n;
  116. signal(SIGWINCH, SIG_IGN);
  117. while(1){
  118. n = io_getevents(ctx, 1, 1, &event, NULL);
  119. if(n < 0){
  120. if(errno == EINTR)
  121. continue;
  122. printk("aio_thread - io_getevents failed, "
  123. "errno = %d\n", errno);
  124. }
  125. else {
  126. /* This is safe as we've just a pointer here. */
  127. aio = (struct aio_context *) (long) event.data;
  128. if(update_aio(aio, event.res)){
  129. do_aio(ctx, aio);
  130. continue;
  131. }
  132. reply = ((struct aio_thread_reply)
  133. { .data = aio,
  134. .err = aio->len });
  135. err = os_write_file(aio->reply_fd, &reply,
  136. sizeof(reply));
  137. if(err != sizeof(reply))
  138. printk("aio_thread - write failed, "
  139. "fd = %d, err = %d\n", aio->reply_fd,
  140. -err);
  141. }
  142. }
  143. return 0;
  144. }
  145. #endif
  146. static int do_not_aio(struct aio_context *aio)
  147. {
  148. char c;
  149. int err;
  150. switch(aio->type){
  151. case AIO_READ:
  152. err = os_seek_file(aio->fd, aio->offset);
  153. if(err)
  154. goto out;
  155. err = os_read_file(aio->fd, aio->data, aio->len);
  156. break;
  157. case AIO_WRITE:
  158. err = os_seek_file(aio->fd, aio->offset);
  159. if(err)
  160. goto out;
  161. err = os_write_file(aio->fd, aio->data, aio->len);
  162. break;
  163. case AIO_MMAP:
  164. err = os_seek_file(aio->fd, aio->offset);
  165. if(err)
  166. goto out;
  167. err = os_read_file(aio->fd, &c, sizeof(c));
  168. break;
  169. default:
  170. printk("do_not_aio - bad request type : %d\n", aio->type);
  171. err = -EINVAL;
  172. break;
  173. }
  174. out:
  175. return err;
  176. }
  177. static int not_aio_thread(void *arg)
  178. {
  179. struct aio_context *aio;
  180. struct aio_thread_reply reply;
  181. int err;
  182. signal(SIGWINCH, SIG_IGN);
  183. while(1){
  184. err = os_read_file(aio_req_fd_r, &aio, sizeof(aio));
  185. if(err != sizeof(aio)){
  186. if(err < 0)
  187. printk("not_aio_thread - read failed, "
  188. "fd = %d, err = %d\n", aio_req_fd_r,
  189. -err);
  190. else {
  191. printk("not_aio_thread - short read, fd = %d, "
  192. "length = %d\n", aio_req_fd_r, err);
  193. }
  194. continue;
  195. }
  196. again:
  197. err = do_not_aio(aio);
  198. if(update_aio(aio, err))
  199. goto again;
  200. reply = ((struct aio_thread_reply) { .data = aio,
  201. .err = aio->len });
  202. err = os_write_file(aio->reply_fd, &reply, sizeof(reply));
  203. if(err != sizeof(reply))
  204. printk("not_aio_thread - write failed, fd = %d, "
  205. "err = %d\n", aio_req_fd_r, -err);
  206. }
  207. }
  208. static int submit_aio_24(struct aio_context *aio)
  209. {
  210. int err;
  211. err = os_write_file(aio_req_fd_w, &aio, sizeof(aio));
  212. if(err == sizeof(aio))
  213. err = 0;
  214. return err;
  215. }
  216. static int aio_pid = -1;
  217. static int (*submit_proc)(struct aio_context *aio);
  218. static int init_aio_24(void)
  219. {
  220. unsigned long stack;
  221. int fds[2], err;
  222. err = os_pipe(fds, 1, 1);
  223. if(err)
  224. goto out;
  225. aio_req_fd_w = fds[0];
  226. aio_req_fd_r = fds[1];
  227. err = run_helper_thread(not_aio_thread, NULL,
  228. CLONE_FILES | CLONE_VM | SIGCHLD, &stack, 0);
  229. if(err < 0)
  230. goto out_close_pipe;
  231. aio_pid = err;
  232. goto out;
  233. out_close_pipe:
  234. os_close_file(fds[0]);
  235. os_close_file(fds[1]);
  236. aio_req_fd_w = -1;
  237. aio_req_fd_r = -1;
  238. out:
  239. #ifndef HAVE_AIO_ABI
  240. printk("/usr/include/linux/aio_abi.h not present during build\n");
  241. #endif
  242. printk("2.6 host AIO support not used - falling back to I/O "
  243. "thread\n");
  244. submit_proc = submit_aio_24;
  245. return 0;
  246. }
  247. #ifdef HAVE_AIO_ABI
  248. #define DEFAULT_24_AIO 0
  249. static int submit_aio_26(struct aio_context *aio)
  250. {
  251. struct aio_thread_reply reply;
  252. int err;
  253. err = do_aio(ctx, aio);
  254. if(err){
  255. reply = ((struct aio_thread_reply) { .data = aio,
  256. .err = err });
  257. err = os_write_file(aio->reply_fd, &reply, sizeof(reply));
  258. if(err != sizeof(reply))
  259. printk("submit_aio_26 - write failed, "
  260. "fd = %d, err = %d\n", aio->reply_fd, -err);
  261. else err = 0;
  262. }
  263. return err;
  264. }
  265. static int init_aio_26(void)
  266. {
  267. unsigned long stack;
  268. int err;
  269. if(io_setup(256, &ctx)){
  270. err = -errno;
  271. printk("aio_thread failed to initialize context, err = %d\n",
  272. errno);
  273. return err;
  274. }
  275. err = run_helper_thread(aio_thread, NULL,
  276. CLONE_FILES | CLONE_VM | SIGCHLD, &stack, 0);
  277. if(err < 0)
  278. return err;
  279. aio_pid = err;
  280. printk("Using 2.6 host AIO\n");
  281. submit_proc = submit_aio_26;
  282. return 0;
  283. }
  284. #else
  285. #define DEFAULT_24_AIO 1
  286. static int submit_aio_26(struct aio_context *aio)
  287. {
  288. return -ENOSYS;
  289. }
  290. static int init_aio_26(void)
  291. {
  292. submit_proc = submit_aio_26;
  293. return -ENOSYS;
  294. }
  295. #endif
  296. static int aio_24 = DEFAULT_24_AIO;
  297. static int __init set_aio_24(char *name, int *add)
  298. {
  299. aio_24 = 1;
  300. return 0;
  301. }
  302. __uml_setup("aio=2.4", set_aio_24,
  303. "aio=2.4\n"
  304. " This is used to force UML to use 2.4-style AIO even when 2.6 AIO is\n"
  305. " available. 2.4 AIO is a single thread that handles one request at a\n"
  306. " time, synchronously. 2.6 AIO is a thread which uses the 2.6 AIO \n"
  307. " interface to handle an arbitrary number of pending requests. 2.6 AIO \n"
  308. " is not available in tt mode, on 2.4 hosts, or when UML is built with\n"
  309. " /usr/include/linux/aio_abi.h not available. Many distributions don't\n"
  310. " include aio_abi.h, so you will need to copy it from a kernel tree to\n"
  311. " your /usr/include/linux in order to build an AIO-capable UML\n\n"
  312. );
  313. static int init_aio(void)
  314. {
  315. int err;
  316. CHOOSE_MODE(({
  317. if(!aio_24){
  318. printk("Disabling 2.6 AIO in tt mode\n");
  319. aio_24 = 1;
  320. } }), (void) 0);
  321. if(!aio_24){
  322. err = init_aio_26();
  323. if(err && (errno == ENOSYS)){
  324. printk("2.6 AIO not supported on the host - "
  325. "reverting to 2.4 AIO\n");
  326. aio_24 = 1;
  327. }
  328. else return err;
  329. }
  330. if(aio_24)
  331. return init_aio_24();
  332. return 0;
  333. }
  334. /* The reason for the __initcall/__uml_exitcall asymmetry is that init_aio
  335. * needs to be called when the kernel is running because it calls run_helper,
  336. * which needs get_free_page. exit_aio is a __uml_exitcall because the generic
  337. * kernel does not run __exitcalls on shutdown, and can't because many of them
  338. * break when called outside of module unloading.
  339. */
  340. __initcall(init_aio);
  341. static void exit_aio(void)
  342. {
  343. if(aio_pid != -1)
  344. os_kill_process(aio_pid, 1);
  345. }
  346. __uml_exitcall(exit_aio);
  347. int submit_aio(struct aio_context *aio)
  348. {
  349. return (*submit_proc)(aio);
  350. }