crypto: arm64/sm3 - new v8.2 Crypto Extensions implementation

Implement the Chinese SM3 secure hash algorithm using the new
special instructions that have been introduced as an optional
extension in ARMv8.2.

Tested-by: Steve Capper <steve.capper@arm.com>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Ard Biesheuvel 2018-01-19 12:04:39 +00:00 committed by Herbert Xu
parent 15d5910e92
commit 140aa50d68
4 changed files with 242 additions and 0 deletions

View File

@ -41,6 +41,12 @@ config CRYPTO_SHA3_ARM64
select CRYPTO_HASH select CRYPTO_HASH
select CRYPTO_SHA3 select CRYPTO_SHA3
config CRYPTO_SM3_ARM64_CE
tristate "SM3 digest algorithm (ARMv8.2 Crypto Extensions)"
depends on KERNEL_MODE_NEON
select CRYPTO_HASH
select CRYPTO_SM3
config CRYPTO_GHASH_ARM64_CE config CRYPTO_GHASH_ARM64_CE
tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions" tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions"
depends on KERNEL_MODE_NEON depends on KERNEL_MODE_NEON

View File

@ -20,6 +20,9 @@ sha512-ce-y := sha512-ce-glue.o sha512-ce-core.o
obj-$(CONFIG_CRYPTO_SHA3_ARM64) += sha3-ce.o obj-$(CONFIG_CRYPTO_SHA3_ARM64) += sha3-ce.o
sha3-ce-y := sha3-ce-glue.o sha3-ce-core.o sha3-ce-y := sha3-ce-glue.o sha3-ce-core.o
obj-$(CONFIG_CRYPTO_SM3_ARM64_CE) += sm3-ce.o
sm3-ce-y := sm3-ce-glue.o sm3-ce-core.o
obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o
ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o

View File

@ -0,0 +1,141 @@
/*
* sm3-ce-core.S - SM3 secure hash using ARMv8.2 Crypto Extensions
*
* Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
.irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
.set .Lv\b\().4s, \b
.endr
.macro sm3partw1, rd, rn, rm
.inst 0xce60c000 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
.endm
.macro sm3partw2, rd, rn, rm
.inst 0xce60c400 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
.endm
.macro sm3ss1, rd, rn, rm, ra
.inst 0xce400000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
.endm
.macro sm3tt1a, rd, rn, rm, imm2
.inst 0xce408000 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
.endm
.macro sm3tt1b, rd, rn, rm, imm2
.inst 0xce408400 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
.endm
.macro sm3tt2a, rd, rn, rm, imm2
.inst 0xce408800 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
.endm
.macro sm3tt2b, rd, rn, rm, imm2
.inst 0xce408c00 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
.endm
.macro round, ab, s0, t0, t1, i
sm3ss1 v5.4s, v8.4s, \t0\().4s, v9.4s
shl \t1\().4s, \t0\().4s, #1
sri \t1\().4s, \t0\().4s, #31
sm3tt1\ab v8.4s, v5.4s, v10.4s, \i
sm3tt2\ab v9.4s, v5.4s, \s0\().4s, \i
.endm
.macro qround, ab, s0, s1, s2, s3, s4
.ifnb \s4
ext \s4\().16b, \s1\().16b, \s2\().16b, #12
ext v6.16b, \s0\().16b, \s1\().16b, #12
ext v7.16b, \s2\().16b, \s3\().16b, #8
sm3partw1 \s4\().4s, \s0\().4s, \s3\().4s
.endif
eor v10.16b, \s0\().16b, \s1\().16b
round \ab, \s0, v11, v12, 0
round \ab, \s0, v12, v11, 1
round \ab, \s0, v11, v12, 2
round \ab, \s0, v12, v11, 3
.ifnb \s4
sm3partw2 \s4\().4s, v7.4s, v6.4s
.endif
.endm
/*
* void sm3_ce_transform(struct sm3_state *sst, u8 const *src,
* int blocks)
*/
.text
ENTRY(sm3_ce_transform)
/* load state */
ld1 {v8.4s-v9.4s}, [x0]
rev64 v8.4s, v8.4s
rev64 v9.4s, v9.4s
ext v8.16b, v8.16b, v8.16b, #8
ext v9.16b, v9.16b, v9.16b, #8
adr_l x8, .Lt
ldp s13, s14, [x8]
/* load input */
0: ld1 {v0.16b-v3.16b}, [x1], #64
sub w2, w2, #1
mov v15.16b, v8.16b
mov v16.16b, v9.16b
CPU_LE( rev32 v0.16b, v0.16b )
CPU_LE( rev32 v1.16b, v1.16b )
CPU_LE( rev32 v2.16b, v2.16b )
CPU_LE( rev32 v3.16b, v3.16b )
ext v11.16b, v13.16b, v13.16b, #4
qround a, v0, v1, v2, v3, v4
qround a, v1, v2, v3, v4, v0
qround a, v2, v3, v4, v0, v1
qround a, v3, v4, v0, v1, v2
ext v11.16b, v14.16b, v14.16b, #4
qround b, v4, v0, v1, v2, v3
qround b, v0, v1, v2, v3, v4
qround b, v1, v2, v3, v4, v0
qround b, v2, v3, v4, v0, v1
qround b, v3, v4, v0, v1, v2
qround b, v4, v0, v1, v2, v3
qround b, v0, v1, v2, v3, v4
qround b, v1, v2, v3, v4, v0
qround b, v2, v3, v4, v0, v1
qround b, v3, v4
qround b, v4, v0
qround b, v0, v1
eor v8.16b, v8.16b, v15.16b
eor v9.16b, v9.16b, v16.16b
/* handled all input blocks? */
cbnz w2, 0b
/* save state */
rev64 v8.4s, v8.4s
rev64 v9.4s, v9.4s
ext v8.16b, v8.16b, v8.16b, #8
ext v9.16b, v9.16b, v9.16b, #8
st1 {v8.4s-v9.4s}, [x0]
ret
ENDPROC(sm3_ce_transform)
.section ".rodata", "a"
.align 3
.Lt: .word 0x79cc4519, 0x9d8a7a87

View File

@ -0,0 +1,92 @@
/*
* sm3-ce-glue.c - SM3 secure hash using ARMv8.2 Crypto Extensions
*
* Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <asm/neon.h>
#include <asm/simd.h>
#include <asm/unaligned.h>
#include <crypto/internal/hash.h>
#include <crypto/sm3.h>
#include <crypto/sm3_base.h>
#include <linux/cpufeature.h>
#include <linux/crypto.h>
#include <linux/module.h>
MODULE_DESCRIPTION("SM3 secure hash using ARMv8 Crypto Extensions");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");
asmlinkage void sm3_ce_transform(struct sm3_state *sst, u8 const *src,
int blocks);
static int sm3_ce_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
if (!may_use_simd())
return crypto_sm3_update(desc, data, len);
kernel_neon_begin();
sm3_base_do_update(desc, data, len, sm3_ce_transform);
kernel_neon_end();
return 0;
}
static int sm3_ce_final(struct shash_desc *desc, u8 *out)
{
if (!may_use_simd())
return crypto_sm3_finup(desc, NULL, 0, out);
kernel_neon_begin();
sm3_base_do_finalize(desc, sm3_ce_transform);
kernel_neon_end();
return sm3_base_finish(desc, out);
}
static int sm3_ce_finup(struct shash_desc *desc, const u8 *data,
unsigned int len, u8 *out)
{
if (!may_use_simd())
return crypto_sm3_finup(desc, data, len, out);
kernel_neon_begin();
sm3_base_do_update(desc, data, len, sm3_ce_transform);
kernel_neon_end();
return sm3_ce_final(desc, out);
}
static struct shash_alg sm3_alg = {
.digestsize = SM3_DIGEST_SIZE,
.init = sm3_base_init,
.update = sm3_ce_update,
.final = sm3_ce_final,
.finup = sm3_ce_finup,
.descsize = sizeof(struct sm3_state),
.base.cra_name = "sm3",
.base.cra_driver_name = "sm3-ce",
.base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
.base.cra_blocksize = SM3_BLOCK_SIZE,
.base.cra_module = THIS_MODULE,
.base.cra_priority = 200,
};
static int __init sm3_ce_mod_init(void)
{
return crypto_register_shash(&sm3_alg);
}
static void __exit sm3_ce_mod_fini(void)
{
crypto_unregister_shash(&sm3_alg);
}
module_cpu_feature_match(SM3, sm3_ce_mod_init);
module_exit(sm3_ce_mod_fini);