cryptd.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622
  1. /*
  2. * Software async crypto daemon.
  3. *
  4. * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License as published by the Free
  8. * Software Foundation; either version 2 of the License, or (at your option)
  9. * any later version.
  10. *
  11. */
  12. #include <crypto/algapi.h>
  13. #include <linux/err.h>
  14. #include <linux/init.h>
  15. #include <linux/kernel.h>
  16. #include <linux/kthread.h>
  17. #include <linux/list.h>
  18. #include <linux/module.h>
  19. #include <linux/mutex.h>
  20. #include <linux/scatterlist.h>
  21. #include <linux/sched.h>
  22. #include <linux/slab.h>
  23. #include <linux/spinlock.h>
  24. #define CRYPTD_MAX_QLEN 100
  25. struct cryptd_state {
  26. spinlock_t lock;
  27. struct mutex mutex;
  28. struct crypto_queue queue;
  29. struct task_struct *task;
  30. };
  31. struct cryptd_instance_ctx {
  32. struct crypto_spawn spawn;
  33. struct cryptd_state *state;
  34. };
  35. struct cryptd_blkcipher_ctx {
  36. struct crypto_blkcipher *child;
  37. };
  38. struct cryptd_blkcipher_request_ctx {
  39. crypto_completion_t complete;
  40. };
  41. struct cryptd_hash_ctx {
  42. struct crypto_hash *child;
  43. };
  44. struct cryptd_hash_request_ctx {
  45. crypto_completion_t complete;
  46. };
  47. static inline struct cryptd_state *cryptd_get_state(struct crypto_tfm *tfm)
  48. {
  49. struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
  50. struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
  51. return ictx->state;
  52. }
  53. static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent,
  54. const u8 *key, unsigned int keylen)
  55. {
  56. struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent);
  57. struct crypto_blkcipher *child = ctx->child;
  58. int err;
  59. crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
  60. crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) &
  61. CRYPTO_TFM_REQ_MASK);
  62. err = crypto_blkcipher_setkey(child, key, keylen);
  63. crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) &
  64. CRYPTO_TFM_RES_MASK);
  65. return err;
  66. }
  67. static void cryptd_blkcipher_crypt(struct ablkcipher_request *req,
  68. struct crypto_blkcipher *child,
  69. int err,
  70. int (*crypt)(struct blkcipher_desc *desc,
  71. struct scatterlist *dst,
  72. struct scatterlist *src,
  73. unsigned int len))
  74. {
  75. struct cryptd_blkcipher_request_ctx *rctx;
  76. struct blkcipher_desc desc;
  77. rctx = ablkcipher_request_ctx(req);
  78. if (unlikely(err == -EINPROGRESS))
  79. goto out;
  80. desc.tfm = child;
  81. desc.info = req->info;
  82. desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
  83. err = crypt(&desc, req->dst, req->src, req->nbytes);
  84. req->base.complete = rctx->complete;
  85. out:
  86. local_bh_disable();
  87. rctx->complete(&req->base, err);
  88. local_bh_enable();
  89. }
  90. static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err)
  91. {
  92. struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
  93. struct crypto_blkcipher *child = ctx->child;
  94. cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
  95. crypto_blkcipher_crt(child)->encrypt);
  96. }
  97. static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err)
  98. {
  99. struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
  100. struct crypto_blkcipher *child = ctx->child;
  101. cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
  102. crypto_blkcipher_crt(child)->decrypt);
  103. }
  104. static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req,
  105. crypto_completion_t complete)
  106. {
  107. struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req);
  108. struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
  109. struct cryptd_state *state =
  110. cryptd_get_state(crypto_ablkcipher_tfm(tfm));
  111. int err;
  112. rctx->complete = req->base.complete;
  113. req->base.complete = complete;
  114. spin_lock_bh(&state->lock);
  115. err = ablkcipher_enqueue_request(&state->queue, req);
  116. spin_unlock_bh(&state->lock);
  117. wake_up_process(state->task);
  118. return err;
  119. }
  120. static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req)
  121. {
  122. return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt);
  123. }
  124. static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req)
  125. {
  126. return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt);
  127. }
  128. static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm)
  129. {
  130. struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
  131. struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
  132. struct crypto_spawn *spawn = &ictx->spawn;
  133. struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
  134. struct crypto_blkcipher *cipher;
  135. cipher = crypto_spawn_blkcipher(spawn);
  136. if (IS_ERR(cipher))
  137. return PTR_ERR(cipher);
  138. ctx->child = cipher;
  139. tfm->crt_ablkcipher.reqsize =
  140. sizeof(struct cryptd_blkcipher_request_ctx);
  141. return 0;
  142. }
  143. static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm)
  144. {
  145. struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
  146. struct cryptd_state *state = cryptd_get_state(tfm);
  147. int active;
  148. mutex_lock(&state->mutex);
  149. active = ablkcipher_tfm_in_queue(&state->queue,
  150. __crypto_ablkcipher_cast(tfm));
  151. mutex_unlock(&state->mutex);
  152. BUG_ON(active);
  153. crypto_free_blkcipher(ctx->child);
  154. }
  155. static struct crypto_instance *cryptd_alloc_instance(struct crypto_alg *alg,
  156. struct cryptd_state *state)
  157. {
  158. struct crypto_instance *inst;
  159. struct cryptd_instance_ctx *ctx;
  160. int err;
  161. inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
  162. if (!inst) {
  163. inst = ERR_PTR(-ENOMEM);
  164. goto out;
  165. }
  166. err = -ENAMETOOLONG;
  167. if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
  168. "cryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
  169. goto out_free_inst;
  170. ctx = crypto_instance_ctx(inst);
  171. err = crypto_init_spawn(&ctx->spawn, alg, inst,
  172. CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC);
  173. if (err)
  174. goto out_free_inst;
  175. ctx->state = state;
  176. memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
  177. inst->alg.cra_priority = alg->cra_priority + 50;
  178. inst->alg.cra_blocksize = alg->cra_blocksize;
  179. inst->alg.cra_alignmask = alg->cra_alignmask;
  180. out:
  181. return inst;
  182. out_free_inst:
  183. kfree(inst);
  184. inst = ERR_PTR(err);
  185. goto out;
  186. }
  187. static struct crypto_instance *cryptd_alloc_blkcipher(
  188. struct rtattr **tb, struct cryptd_state *state)
  189. {
  190. struct crypto_instance *inst;
  191. struct crypto_alg *alg;
  192. alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_BLKCIPHER,
  193. CRYPTO_ALG_TYPE_MASK);
  194. if (IS_ERR(alg))
  195. return ERR_CAST(alg);
  196. inst = cryptd_alloc_instance(alg, state);
  197. if (IS_ERR(inst))
  198. goto out_put_alg;
  199. inst->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
  200. inst->alg.cra_type = &crypto_ablkcipher_type;
  201. inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize;
  202. inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
  203. inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
  204. inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv;
  205. inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx);
  206. inst->alg.cra_init = cryptd_blkcipher_init_tfm;
  207. inst->alg.cra_exit = cryptd_blkcipher_exit_tfm;
  208. inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey;
  209. inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue;
  210. inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue;
  211. out_put_alg:
  212. crypto_mod_put(alg);
  213. return inst;
  214. }
  215. static int cryptd_hash_init_tfm(struct crypto_tfm *tfm)
  216. {
  217. struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
  218. struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
  219. struct crypto_spawn *spawn = &ictx->spawn;
  220. struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
  221. struct crypto_hash *cipher;
  222. cipher = crypto_spawn_hash(spawn);
  223. if (IS_ERR(cipher))
  224. return PTR_ERR(cipher);
  225. ctx->child = cipher;
  226. tfm->crt_ahash.reqsize =
  227. sizeof(struct cryptd_hash_request_ctx);
  228. return 0;
  229. }
  230. static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm)
  231. {
  232. struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
  233. struct cryptd_state *state = cryptd_get_state(tfm);
  234. int active;
  235. mutex_lock(&state->mutex);
  236. active = ahash_tfm_in_queue(&state->queue,
  237. __crypto_ahash_cast(tfm));
  238. mutex_unlock(&state->mutex);
  239. BUG_ON(active);
  240. crypto_free_hash(ctx->child);
  241. }
  242. static int cryptd_hash_setkey(struct crypto_ahash *parent,
  243. const u8 *key, unsigned int keylen)
  244. {
  245. struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
  246. struct crypto_hash *child = ctx->child;
  247. int err;
  248. crypto_hash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
  249. crypto_hash_set_flags(child, crypto_ahash_get_flags(parent) &
  250. CRYPTO_TFM_REQ_MASK);
  251. err = crypto_hash_setkey(child, key, keylen);
  252. crypto_ahash_set_flags(parent, crypto_hash_get_flags(child) &
  253. CRYPTO_TFM_RES_MASK);
  254. return err;
  255. }
  256. static int cryptd_hash_enqueue(struct ahash_request *req,
  257. crypto_completion_t complete)
  258. {
  259. struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
  260. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  261. struct cryptd_state *state =
  262. cryptd_get_state(crypto_ahash_tfm(tfm));
  263. int err;
  264. rctx->complete = req->base.complete;
  265. req->base.complete = complete;
  266. spin_lock_bh(&state->lock);
  267. err = ahash_enqueue_request(&state->queue, req);
  268. spin_unlock_bh(&state->lock);
  269. wake_up_process(state->task);
  270. return err;
  271. }
  272. static void cryptd_hash_init(struct crypto_async_request *req_async, int err)
  273. {
  274. struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
  275. struct crypto_hash *child = ctx->child;
  276. struct ahash_request *req = ahash_request_cast(req_async);
  277. struct cryptd_hash_request_ctx *rctx;
  278. struct hash_desc desc;
  279. rctx = ahash_request_ctx(req);
  280. if (unlikely(err == -EINPROGRESS))
  281. goto out;
  282. desc.tfm = child;
  283. desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
  284. err = crypto_hash_crt(child)->init(&desc);
  285. req->base.complete = rctx->complete;
  286. out:
  287. local_bh_disable();
  288. rctx->complete(&req->base, err);
  289. local_bh_enable();
  290. }
  291. static int cryptd_hash_init_enqueue(struct ahash_request *req)
  292. {
  293. return cryptd_hash_enqueue(req, cryptd_hash_init);
  294. }
  295. static void cryptd_hash_update(struct crypto_async_request *req_async, int err)
  296. {
  297. struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
  298. struct crypto_hash *child = ctx->child;
  299. struct ahash_request *req = ahash_request_cast(req_async);
  300. struct cryptd_hash_request_ctx *rctx;
  301. struct hash_desc desc;
  302. rctx = ahash_request_ctx(req);
  303. if (unlikely(err == -EINPROGRESS))
  304. goto out;
  305. desc.tfm = child;
  306. desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
  307. err = crypto_hash_crt(child)->update(&desc,
  308. req->src,
  309. req->nbytes);
  310. req->base.complete = rctx->complete;
  311. out:
  312. local_bh_disable();
  313. rctx->complete(&req->base, err);
  314. local_bh_enable();
  315. }
  316. static int cryptd_hash_update_enqueue(struct ahash_request *req)
  317. {
  318. return cryptd_hash_enqueue(req, cryptd_hash_update);
  319. }
  320. static void cryptd_hash_final(struct crypto_async_request *req_async, int err)
  321. {
  322. struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
  323. struct crypto_hash *child = ctx->child;
  324. struct ahash_request *req = ahash_request_cast(req_async);
  325. struct cryptd_hash_request_ctx *rctx;
  326. struct hash_desc desc;
  327. rctx = ahash_request_ctx(req);
  328. if (unlikely(err == -EINPROGRESS))
  329. goto out;
  330. desc.tfm = child;
  331. desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
  332. err = crypto_hash_crt(child)->final(&desc, req->result);
  333. req->base.complete = rctx->complete;
  334. out:
  335. local_bh_disable();
  336. rctx->complete(&req->base, err);
  337. local_bh_enable();
  338. }
  339. static int cryptd_hash_final_enqueue(struct ahash_request *req)
  340. {
  341. return cryptd_hash_enqueue(req, cryptd_hash_final);
  342. }
  343. static void cryptd_hash_digest(struct crypto_async_request *req_async, int err)
  344. {
  345. struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
  346. struct crypto_hash *child = ctx->child;
  347. struct ahash_request *req = ahash_request_cast(req_async);
  348. struct cryptd_hash_request_ctx *rctx;
  349. struct hash_desc desc;
  350. rctx = ahash_request_ctx(req);
  351. if (unlikely(err == -EINPROGRESS))
  352. goto out;
  353. desc.tfm = child;
  354. desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
  355. err = crypto_hash_crt(child)->digest(&desc,
  356. req->src,
  357. req->nbytes,
  358. req->result);
  359. req->base.complete = rctx->complete;
  360. out:
  361. local_bh_disable();
  362. rctx->complete(&req->base, err);
  363. local_bh_enable();
  364. }
  365. static int cryptd_hash_digest_enqueue(struct ahash_request *req)
  366. {
  367. return cryptd_hash_enqueue(req, cryptd_hash_digest);
  368. }
  369. static struct crypto_instance *cryptd_alloc_hash(
  370. struct rtattr **tb, struct cryptd_state *state)
  371. {
  372. struct crypto_instance *inst;
  373. struct crypto_alg *alg;
  374. alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_HASH,
  375. CRYPTO_ALG_TYPE_HASH_MASK);
  376. if (IS_ERR(alg))
  377. return ERR_PTR(PTR_ERR(alg));
  378. inst = cryptd_alloc_instance(alg, state);
  379. if (IS_ERR(inst))
  380. goto out_put_alg;
  381. inst->alg.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC;
  382. inst->alg.cra_type = &crypto_ahash_type;
  383. inst->alg.cra_ahash.digestsize = alg->cra_hash.digestsize;
  384. inst->alg.cra_ctxsize = sizeof(struct cryptd_hash_ctx);
  385. inst->alg.cra_init = cryptd_hash_init_tfm;
  386. inst->alg.cra_exit = cryptd_hash_exit_tfm;
  387. inst->alg.cra_ahash.init = cryptd_hash_init_enqueue;
  388. inst->alg.cra_ahash.update = cryptd_hash_update_enqueue;
  389. inst->alg.cra_ahash.final = cryptd_hash_final_enqueue;
  390. inst->alg.cra_ahash.setkey = cryptd_hash_setkey;
  391. inst->alg.cra_ahash.digest = cryptd_hash_digest_enqueue;
  392. out_put_alg:
  393. crypto_mod_put(alg);
  394. return inst;
  395. }
  396. static struct cryptd_state state;
  397. static struct crypto_instance *cryptd_alloc(struct rtattr **tb)
  398. {
  399. struct crypto_attr_type *algt;
  400. algt = crypto_get_attr_type(tb);
  401. if (IS_ERR(algt))
  402. return ERR_CAST(algt);
  403. switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
  404. case CRYPTO_ALG_TYPE_BLKCIPHER:
  405. return cryptd_alloc_blkcipher(tb, &state);
  406. case CRYPTO_ALG_TYPE_DIGEST:
  407. return cryptd_alloc_hash(tb, &state);
  408. }
  409. return ERR_PTR(-EINVAL);
  410. }
  411. static void cryptd_free(struct crypto_instance *inst)
  412. {
  413. struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
  414. crypto_drop_spawn(&ctx->spawn);
  415. kfree(inst);
  416. }
  417. static struct crypto_template cryptd_tmpl = {
  418. .name = "cryptd",
  419. .alloc = cryptd_alloc,
  420. .free = cryptd_free,
  421. .module = THIS_MODULE,
  422. };
  423. static inline int cryptd_create_thread(struct cryptd_state *state,
  424. int (*fn)(void *data), const char *name)
  425. {
  426. spin_lock_init(&state->lock);
  427. mutex_init(&state->mutex);
  428. crypto_init_queue(&state->queue, CRYPTD_MAX_QLEN);
  429. state->task = kthread_run(fn, state, name);
  430. if (IS_ERR(state->task))
  431. return PTR_ERR(state->task);
  432. return 0;
  433. }
  434. static inline void cryptd_stop_thread(struct cryptd_state *state)
  435. {
  436. BUG_ON(state->queue.qlen);
  437. kthread_stop(state->task);
  438. }
  439. static int cryptd_thread(void *data)
  440. {
  441. struct cryptd_state *state = data;
  442. int stop;
  443. current->flags |= PF_NOFREEZE;
  444. do {
  445. struct crypto_async_request *req, *backlog;
  446. mutex_lock(&state->mutex);
  447. __set_current_state(TASK_INTERRUPTIBLE);
  448. spin_lock_bh(&state->lock);
  449. backlog = crypto_get_backlog(&state->queue);
  450. req = crypto_dequeue_request(&state->queue);
  451. spin_unlock_bh(&state->lock);
  452. stop = kthread_should_stop();
  453. if (stop || req) {
  454. __set_current_state(TASK_RUNNING);
  455. if (req) {
  456. if (backlog)
  457. backlog->complete(backlog,
  458. -EINPROGRESS);
  459. req->complete(req, 0);
  460. }
  461. }
  462. mutex_unlock(&state->mutex);
  463. schedule();
  464. } while (!stop);
  465. return 0;
  466. }
  467. static int __init cryptd_init(void)
  468. {
  469. int err;
  470. err = cryptd_create_thread(&state, cryptd_thread, "cryptd");
  471. if (err)
  472. return err;
  473. err = crypto_register_template(&cryptd_tmpl);
  474. if (err)
  475. kthread_stop(state.task);
  476. return err;
  477. }
  478. static void __exit cryptd_exit(void)
  479. {
  480. cryptd_stop_thread(&state);
  481. crypto_unregister_template(&cryptd_tmpl);
  482. }
  483. module_init(cryptd_init);
  484. module_exit(cryptd_exit);
  485. MODULE_LICENSE("GPL");
  486. MODULE_DESCRIPTION("Software async crypto daemon");