cast6_avx_glue.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614
  1. /*
  2. * Glue Code for the AVX assembler implemention of the Cast6 Cipher
  3. *
  4. * Copyright (C) 2012 Johannes Goetzfried
  5. * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
  6. *
  7. * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
  8. *
  9. * This program is free software; you can redistribute it and/or modify
  10. * it under the terms of the GNU General Public License as published by
  11. * the Free Software Foundation; either version 2 of the License, or
  12. * (at your option) any later version.
  13. *
  14. * This program is distributed in the hope that it will be useful,
  15. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17. * GNU General Public License for more details.
  18. *
  19. * You should have received a copy of the GNU General Public License
  20. * along with this program; if not, write to the Free Software
  21. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
  22. * USA
  23. *
  24. */
  25. #include <linux/module.h>
  26. #include <linux/hardirq.h>
  27. #include <linux/types.h>
  28. #include <linux/crypto.h>
  29. #include <linux/err.h>
  30. #include <crypto/algapi.h>
  31. #include <crypto/cast6.h>
  32. #include <crypto/cryptd.h>
  33. #include <crypto/b128ops.h>
  34. #include <crypto/ctr.h>
  35. #include <crypto/lrw.h>
  36. #include <crypto/xts.h>
  37. #include <asm/xcr.h>
  38. #include <asm/xsave.h>
  39. #include <asm/crypto/ablk_helper.h>
  40. #include <asm/crypto/glue_helper.h>
  41. #define CAST6_PARALLEL_BLOCKS 8
  42. asmlinkage void cast6_ecb_enc_8way(struct cast6_ctx *ctx, u8 *dst,
  43. const u8 *src);
  44. asmlinkage void cast6_ecb_dec_8way(struct cast6_ctx *ctx, u8 *dst,
  45. const u8 *src);
  46. asmlinkage void cast6_cbc_dec_8way(struct cast6_ctx *ctx, u8 *dst,
  47. const u8 *src);
  48. asmlinkage void cast6_ctr_8way(struct cast6_ctx *ctx, u8 *dst, const u8 *src,
  49. le128 *iv);
  50. asmlinkage void cast6_xts_enc_8way(struct cast6_ctx *ctx, u8 *dst,
  51. const u8 *src, le128 *iv);
  52. asmlinkage void cast6_xts_dec_8way(struct cast6_ctx *ctx, u8 *dst,
  53. const u8 *src, le128 *iv);
  54. static void cast6_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  55. {
  56. glue_xts_crypt_128bit_one(ctx, dst, src, iv,
  57. GLUE_FUNC_CAST(__cast6_encrypt));
  58. }
  59. static void cast6_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  60. {
  61. glue_xts_crypt_128bit_one(ctx, dst, src, iv,
  62. GLUE_FUNC_CAST(__cast6_decrypt));
  63. }
  64. static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  65. {
  66. be128 ctrblk;
  67. le128_to_be128(&ctrblk, iv);
  68. le128_inc(iv);
  69. __cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
  70. u128_xor(dst, src, (u128 *)&ctrblk);
  71. }
  72. static const struct common_glue_ctx cast6_enc = {
  73. .num_funcs = 2,
  74. .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
  75. .funcs = { {
  76. .num_blocks = CAST6_PARALLEL_BLOCKS,
  77. .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_enc_8way) }
  78. }, {
  79. .num_blocks = 1,
  80. .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_encrypt) }
  81. } }
  82. };
  83. static const struct common_glue_ctx cast6_ctr = {
  84. .num_funcs = 2,
  85. .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
  86. .funcs = { {
  87. .num_blocks = CAST6_PARALLEL_BLOCKS,
  88. .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_ctr_8way) }
  89. }, {
  90. .num_blocks = 1,
  91. .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr) }
  92. } }
  93. };
  94. static const struct common_glue_ctx cast6_enc_xts = {
  95. .num_funcs = 2,
  96. .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
  97. .funcs = { {
  98. .num_blocks = CAST6_PARALLEL_BLOCKS,
  99. .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc_8way) }
  100. }, {
  101. .num_blocks = 1,
  102. .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc) }
  103. } }
  104. };
  105. static const struct common_glue_ctx cast6_dec = {
  106. .num_funcs = 2,
  107. .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
  108. .funcs = { {
  109. .num_blocks = CAST6_PARALLEL_BLOCKS,
  110. .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_dec_8way) }
  111. }, {
  112. .num_blocks = 1,
  113. .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_decrypt) }
  114. } }
  115. };
  116. static const struct common_glue_ctx cast6_dec_cbc = {
  117. .num_funcs = 2,
  118. .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
  119. .funcs = { {
  120. .num_blocks = CAST6_PARALLEL_BLOCKS,
  121. .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_cbc_dec_8way) }
  122. }, {
  123. .num_blocks = 1,
  124. .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__cast6_decrypt) }
  125. } }
  126. };
  127. static const struct common_glue_ctx cast6_dec_xts = {
  128. .num_funcs = 2,
  129. .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
  130. .funcs = { {
  131. .num_blocks = CAST6_PARALLEL_BLOCKS,
  132. .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec_8way) }
  133. }, {
  134. .num_blocks = 1,
  135. .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec) }
  136. } }
  137. };
  138. static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  139. struct scatterlist *src, unsigned int nbytes)
  140. {
  141. return glue_ecb_crypt_128bit(&cast6_enc, desc, dst, src, nbytes);
  142. }
  143. static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  144. struct scatterlist *src, unsigned int nbytes)
  145. {
  146. return glue_ecb_crypt_128bit(&cast6_dec, desc, dst, src, nbytes);
  147. }
  148. static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  149. struct scatterlist *src, unsigned int nbytes)
  150. {
  151. return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__cast6_encrypt), desc,
  152. dst, src, nbytes);
  153. }
  154. static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  155. struct scatterlist *src, unsigned int nbytes)
  156. {
  157. return glue_cbc_decrypt_128bit(&cast6_dec_cbc, desc, dst, src,
  158. nbytes);
  159. }
  160. static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  161. struct scatterlist *src, unsigned int nbytes)
  162. {
  163. return glue_ctr_crypt_128bit(&cast6_ctr, desc, dst, src, nbytes);
  164. }
  165. static inline bool cast6_fpu_begin(bool fpu_enabled, unsigned int nbytes)
  166. {
  167. return glue_fpu_begin(CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS,
  168. NULL, fpu_enabled, nbytes);
  169. }
  170. static inline void cast6_fpu_end(bool fpu_enabled)
  171. {
  172. glue_fpu_end(fpu_enabled);
  173. }
  174. struct crypt_priv {
  175. struct cast6_ctx *ctx;
  176. bool fpu_enabled;
  177. };
  178. static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
  179. {
  180. const unsigned int bsize = CAST6_BLOCK_SIZE;
  181. struct crypt_priv *ctx = priv;
  182. int i;
  183. ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
  184. if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
  185. cast6_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
  186. return;
  187. }
  188. for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
  189. __cast6_encrypt(ctx->ctx, srcdst, srcdst);
  190. }
  191. static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
  192. {
  193. const unsigned int bsize = CAST6_BLOCK_SIZE;
  194. struct crypt_priv *ctx = priv;
  195. int i;
  196. ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
  197. if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
  198. cast6_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
  199. return;
  200. }
  201. for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
  202. __cast6_decrypt(ctx->ctx, srcdst, srcdst);
  203. }
  204. struct cast6_lrw_ctx {
  205. struct lrw_table_ctx lrw_table;
  206. struct cast6_ctx cast6_ctx;
  207. };
  208. static int lrw_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
  209. unsigned int keylen)
  210. {
  211. struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
  212. int err;
  213. err = __cast6_setkey(&ctx->cast6_ctx, key, keylen - CAST6_BLOCK_SIZE,
  214. &tfm->crt_flags);
  215. if (err)
  216. return err;
  217. return lrw_init_table(&ctx->lrw_table, key + keylen - CAST6_BLOCK_SIZE);
  218. }
  219. static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  220. struct scatterlist *src, unsigned int nbytes)
  221. {
  222. struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  223. be128 buf[CAST6_PARALLEL_BLOCKS];
  224. struct crypt_priv crypt_ctx = {
  225. .ctx = &ctx->cast6_ctx,
  226. .fpu_enabled = false,
  227. };
  228. struct lrw_crypt_req req = {
  229. .tbuf = buf,
  230. .tbuflen = sizeof(buf),
  231. .table_ctx = &ctx->lrw_table,
  232. .crypt_ctx = &crypt_ctx,
  233. .crypt_fn = encrypt_callback,
  234. };
  235. int ret;
  236. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  237. ret = lrw_crypt(desc, dst, src, nbytes, &req);
  238. cast6_fpu_end(crypt_ctx.fpu_enabled);
  239. return ret;
  240. }
  241. static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  242. struct scatterlist *src, unsigned int nbytes)
  243. {
  244. struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  245. be128 buf[CAST6_PARALLEL_BLOCKS];
  246. struct crypt_priv crypt_ctx = {
  247. .ctx = &ctx->cast6_ctx,
  248. .fpu_enabled = false,
  249. };
  250. struct lrw_crypt_req req = {
  251. .tbuf = buf,
  252. .tbuflen = sizeof(buf),
  253. .table_ctx = &ctx->lrw_table,
  254. .crypt_ctx = &crypt_ctx,
  255. .crypt_fn = decrypt_callback,
  256. };
  257. int ret;
  258. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  259. ret = lrw_crypt(desc, dst, src, nbytes, &req);
  260. cast6_fpu_end(crypt_ctx.fpu_enabled);
  261. return ret;
  262. }
  263. static void lrw_exit_tfm(struct crypto_tfm *tfm)
  264. {
  265. struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
  266. lrw_free_table(&ctx->lrw_table);
  267. }
  268. struct cast6_xts_ctx {
  269. struct cast6_ctx tweak_ctx;
  270. struct cast6_ctx crypt_ctx;
  271. };
  272. static int xts_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
  273. unsigned int keylen)
  274. {
  275. struct cast6_xts_ctx *ctx = crypto_tfm_ctx(tfm);
  276. u32 *flags = &tfm->crt_flags;
  277. int err;
  278. /* key consists of keys of equal size concatenated, therefore
  279. * the length must be even
  280. */
  281. if (keylen % 2) {
  282. *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
  283. return -EINVAL;
  284. }
  285. /* first half of xts-key is for crypt */
  286. err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
  287. if (err)
  288. return err;
  289. /* second half of xts-key is for tweak */
  290. return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
  291. flags);
  292. }
  293. static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  294. struct scatterlist *src, unsigned int nbytes)
  295. {
  296. struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  297. return glue_xts_crypt_128bit(&cast6_enc_xts, desc, dst, src, nbytes,
  298. XTS_TWEAK_CAST(__cast6_encrypt),
  299. &ctx->tweak_ctx, &ctx->crypt_ctx);
  300. }
  301. static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  302. struct scatterlist *src, unsigned int nbytes)
  303. {
  304. struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  305. return glue_xts_crypt_128bit(&cast6_dec_xts, desc, dst, src, nbytes,
  306. XTS_TWEAK_CAST(__cast6_encrypt),
  307. &ctx->tweak_ctx, &ctx->crypt_ctx);
  308. }
  309. static struct crypto_alg cast6_algs[10] = { {
  310. .cra_name = "__ecb-cast6-avx",
  311. .cra_driver_name = "__driver-ecb-cast6-avx",
  312. .cra_priority = 0,
  313. .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
  314. .cra_blocksize = CAST6_BLOCK_SIZE,
  315. .cra_ctxsize = sizeof(struct cast6_ctx),
  316. .cra_alignmask = 0,
  317. .cra_type = &crypto_blkcipher_type,
  318. .cra_module = THIS_MODULE,
  319. .cra_u = {
  320. .blkcipher = {
  321. .min_keysize = CAST6_MIN_KEY_SIZE,
  322. .max_keysize = CAST6_MAX_KEY_SIZE,
  323. .setkey = cast6_setkey,
  324. .encrypt = ecb_encrypt,
  325. .decrypt = ecb_decrypt,
  326. },
  327. },
  328. }, {
  329. .cra_name = "__cbc-cast6-avx",
  330. .cra_driver_name = "__driver-cbc-cast6-avx",
  331. .cra_priority = 0,
  332. .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
  333. .cra_blocksize = CAST6_BLOCK_SIZE,
  334. .cra_ctxsize = sizeof(struct cast6_ctx),
  335. .cra_alignmask = 0,
  336. .cra_type = &crypto_blkcipher_type,
  337. .cra_module = THIS_MODULE,
  338. .cra_u = {
  339. .blkcipher = {
  340. .min_keysize = CAST6_MIN_KEY_SIZE,
  341. .max_keysize = CAST6_MAX_KEY_SIZE,
  342. .setkey = cast6_setkey,
  343. .encrypt = cbc_encrypt,
  344. .decrypt = cbc_decrypt,
  345. },
  346. },
  347. }, {
  348. .cra_name = "__ctr-cast6-avx",
  349. .cra_driver_name = "__driver-ctr-cast6-avx",
  350. .cra_priority = 0,
  351. .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
  352. .cra_blocksize = 1,
  353. .cra_ctxsize = sizeof(struct cast6_ctx),
  354. .cra_alignmask = 0,
  355. .cra_type = &crypto_blkcipher_type,
  356. .cra_module = THIS_MODULE,
  357. .cra_u = {
  358. .blkcipher = {
  359. .min_keysize = CAST6_MIN_KEY_SIZE,
  360. .max_keysize = CAST6_MAX_KEY_SIZE,
  361. .ivsize = CAST6_BLOCK_SIZE,
  362. .setkey = cast6_setkey,
  363. .encrypt = ctr_crypt,
  364. .decrypt = ctr_crypt,
  365. },
  366. },
  367. }, {
  368. .cra_name = "__lrw-cast6-avx",
  369. .cra_driver_name = "__driver-lrw-cast6-avx",
  370. .cra_priority = 0,
  371. .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
  372. .cra_blocksize = CAST6_BLOCK_SIZE,
  373. .cra_ctxsize = sizeof(struct cast6_lrw_ctx),
  374. .cra_alignmask = 0,
  375. .cra_type = &crypto_blkcipher_type,
  376. .cra_module = THIS_MODULE,
  377. .cra_exit = lrw_exit_tfm,
  378. .cra_u = {
  379. .blkcipher = {
  380. .min_keysize = CAST6_MIN_KEY_SIZE +
  381. CAST6_BLOCK_SIZE,
  382. .max_keysize = CAST6_MAX_KEY_SIZE +
  383. CAST6_BLOCK_SIZE,
  384. .ivsize = CAST6_BLOCK_SIZE,
  385. .setkey = lrw_cast6_setkey,
  386. .encrypt = lrw_encrypt,
  387. .decrypt = lrw_decrypt,
  388. },
  389. },
  390. }, {
  391. .cra_name = "__xts-cast6-avx",
  392. .cra_driver_name = "__driver-xts-cast6-avx",
  393. .cra_priority = 0,
  394. .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
  395. .cra_blocksize = CAST6_BLOCK_SIZE,
  396. .cra_ctxsize = sizeof(struct cast6_xts_ctx),
  397. .cra_alignmask = 0,
  398. .cra_type = &crypto_blkcipher_type,
  399. .cra_module = THIS_MODULE,
  400. .cra_u = {
  401. .blkcipher = {
  402. .min_keysize = CAST6_MIN_KEY_SIZE * 2,
  403. .max_keysize = CAST6_MAX_KEY_SIZE * 2,
  404. .ivsize = CAST6_BLOCK_SIZE,
  405. .setkey = xts_cast6_setkey,
  406. .encrypt = xts_encrypt,
  407. .decrypt = xts_decrypt,
  408. },
  409. },
  410. }, {
  411. .cra_name = "ecb(cast6)",
  412. .cra_driver_name = "ecb-cast6-avx",
  413. .cra_priority = 200,
  414. .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
  415. .cra_blocksize = CAST6_BLOCK_SIZE,
  416. .cra_ctxsize = sizeof(struct async_helper_ctx),
  417. .cra_alignmask = 0,
  418. .cra_type = &crypto_ablkcipher_type,
  419. .cra_module = THIS_MODULE,
  420. .cra_init = ablk_init,
  421. .cra_exit = ablk_exit,
  422. .cra_u = {
  423. .ablkcipher = {
  424. .min_keysize = CAST6_MIN_KEY_SIZE,
  425. .max_keysize = CAST6_MAX_KEY_SIZE,
  426. .setkey = ablk_set_key,
  427. .encrypt = ablk_encrypt,
  428. .decrypt = ablk_decrypt,
  429. },
  430. },
  431. }, {
  432. .cra_name = "cbc(cast6)",
  433. .cra_driver_name = "cbc-cast6-avx",
  434. .cra_priority = 200,
  435. .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
  436. .cra_blocksize = CAST6_BLOCK_SIZE,
  437. .cra_ctxsize = sizeof(struct async_helper_ctx),
  438. .cra_alignmask = 0,
  439. .cra_type = &crypto_ablkcipher_type,
  440. .cra_module = THIS_MODULE,
  441. .cra_init = ablk_init,
  442. .cra_exit = ablk_exit,
  443. .cra_u = {
  444. .ablkcipher = {
  445. .min_keysize = CAST6_MIN_KEY_SIZE,
  446. .max_keysize = CAST6_MAX_KEY_SIZE,
  447. .ivsize = CAST6_BLOCK_SIZE,
  448. .setkey = ablk_set_key,
  449. .encrypt = __ablk_encrypt,
  450. .decrypt = ablk_decrypt,
  451. },
  452. },
  453. }, {
  454. .cra_name = "ctr(cast6)",
  455. .cra_driver_name = "ctr-cast6-avx",
  456. .cra_priority = 200,
  457. .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
  458. .cra_blocksize = 1,
  459. .cra_ctxsize = sizeof(struct async_helper_ctx),
  460. .cra_alignmask = 0,
  461. .cra_type = &crypto_ablkcipher_type,
  462. .cra_module = THIS_MODULE,
  463. .cra_init = ablk_init,
  464. .cra_exit = ablk_exit,
  465. .cra_u = {
  466. .ablkcipher = {
  467. .min_keysize = CAST6_MIN_KEY_SIZE,
  468. .max_keysize = CAST6_MAX_KEY_SIZE,
  469. .ivsize = CAST6_BLOCK_SIZE,
  470. .setkey = ablk_set_key,
  471. .encrypt = ablk_encrypt,
  472. .decrypt = ablk_encrypt,
  473. .geniv = "chainiv",
  474. },
  475. },
  476. }, {
  477. .cra_name = "lrw(cast6)",
  478. .cra_driver_name = "lrw-cast6-avx",
  479. .cra_priority = 200,
  480. .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
  481. .cra_blocksize = CAST6_BLOCK_SIZE,
  482. .cra_ctxsize = sizeof(struct async_helper_ctx),
  483. .cra_alignmask = 0,
  484. .cra_type = &crypto_ablkcipher_type,
  485. .cra_module = THIS_MODULE,
  486. .cra_init = ablk_init,
  487. .cra_exit = ablk_exit,
  488. .cra_u = {
  489. .ablkcipher = {
  490. .min_keysize = CAST6_MIN_KEY_SIZE +
  491. CAST6_BLOCK_SIZE,
  492. .max_keysize = CAST6_MAX_KEY_SIZE +
  493. CAST6_BLOCK_SIZE,
  494. .ivsize = CAST6_BLOCK_SIZE,
  495. .setkey = ablk_set_key,
  496. .encrypt = ablk_encrypt,
  497. .decrypt = ablk_decrypt,
  498. },
  499. },
  500. }, {
  501. .cra_name = "xts(cast6)",
  502. .cra_driver_name = "xts-cast6-avx",
  503. .cra_priority = 200,
  504. .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
  505. .cra_blocksize = CAST6_BLOCK_SIZE,
  506. .cra_ctxsize = sizeof(struct async_helper_ctx),
  507. .cra_alignmask = 0,
  508. .cra_type = &crypto_ablkcipher_type,
  509. .cra_module = THIS_MODULE,
  510. .cra_init = ablk_init,
  511. .cra_exit = ablk_exit,
  512. .cra_u = {
  513. .ablkcipher = {
  514. .min_keysize = CAST6_MIN_KEY_SIZE * 2,
  515. .max_keysize = CAST6_MAX_KEY_SIZE * 2,
  516. .ivsize = CAST6_BLOCK_SIZE,
  517. .setkey = ablk_set_key,
  518. .encrypt = ablk_encrypt,
  519. .decrypt = ablk_decrypt,
  520. },
  521. },
  522. } };
  523. static int __init cast6_init(void)
  524. {
  525. u64 xcr0;
  526. if (!cpu_has_avx || !cpu_has_osxsave) {
  527. pr_info("AVX instructions are not detected.\n");
  528. return -ENODEV;
  529. }
  530. xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
  531. if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
  532. pr_info("AVX detected but unusable.\n");
  533. return -ENODEV;
  534. }
  535. return crypto_register_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
  536. }
  537. static void __exit cast6_exit(void)
  538. {
  539. crypto_unregister_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
  540. }
  541. module_init(cast6_init);
  542. module_exit(cast6_exit);
  543. MODULE_DESCRIPTION("Cast6 Cipher Algorithm, AVX optimized");
  544. MODULE_LICENSE("GPL");
  545. MODULE_ALIAS("cast6");