diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig index f755da979534..c1932ce46c7f 100644 --- a/arch/sparc/crypto/Kconfig +++ b/arch/sparc/crypto/Kconfig @@ -19,9 +19,9 @@ config CRYPTO_DES_SPARC64 config CRYPTO_AES_SPARC64 tristate "Ciphers: AES, modes: ECB, CBC, CTR" depends on SPARC64 + select CRYPTO_LIB_AES select CRYPTO_SKCIPHER help - Block ciphers: AES cipher algorithms (FIPS-197) Length-preseving ciphers: AES with ECB, CBC, and CTR modes Architecture: sparc64 using crypto instructions diff --git a/arch/sparc/crypto/Makefile b/arch/sparc/crypto/Makefile index 7b4796842ddd..cdf9f4b3efbb 100644 --- a/arch/sparc/crypto/Makefile +++ b/arch/sparc/crypto/Makefile @@ -7,6 +7,6 @@ obj-$(CONFIG_CRYPTO_AES_SPARC64) += aes-sparc64.o obj-$(CONFIG_CRYPTO_DES_SPARC64) += des-sparc64.o obj-$(CONFIG_CRYPTO_CAMELLIA_SPARC64) += camellia-sparc64.o -aes-sparc64-y := aes_asm.o aes_glue.o +aes-sparc64-y := aes_glue.o des-sparc64-y := des_asm.o des_glue.o camellia-sparc64-y := camellia_asm.o camellia_glue.o diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c index 359f22643b05..661561837415 100644 --- a/arch/sparc/crypto/aes_glue.c +++ b/arch/sparc/crypto/aes_glue.c @@ -32,8 +32,6 @@ #include struct aes_ops { - void (*encrypt)(const u64 *key, const u32 *input, u32 *output); - void (*decrypt)(const u64 *key, const u32 *input, u32 *output); void (*load_encrypt_keys)(const u64 *key); void (*load_decrypt_keys)(const u64 *key); void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output, @@ -55,79 +53,7 @@ struct crypto_sparc64_aes_ctx { u32 expanded_key_length; }; -extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input, - u32 *output); -extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input, - u32 *output); -extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input, - u32 *output); - -extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input, - u32 *output); -extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input, - u32 *output); -extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input, - u32 *output); - -extern void aes_sparc64_load_encrypt_keys_128(const u64 *key); -extern void aes_sparc64_load_encrypt_keys_192(const u64 *key); -extern void aes_sparc64_load_encrypt_keys_256(const u64 *key); - -extern void aes_sparc64_load_decrypt_keys_128(const u64 *key); -extern void aes_sparc64_load_decrypt_keys_192(const u64 *key); -extern void aes_sparc64_load_decrypt_keys_256(const u64 *key); - -extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input, - u64 *output, unsigned int len); -extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input, - u64 *output, unsigned int len); -extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input, - u64 *output, unsigned int len); - -extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input, - u64 *output, unsigned int len); -extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input, - u64 *output, unsigned int len); -extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input, - u64 *output, unsigned int len); - -extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input, - u64 *output, unsigned int len, - u64 *iv); - -extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input, - u64 *output, unsigned int len, - u64 *iv); - -extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input, - u64 *output, unsigned int len, - u64 *iv); - -extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input, - u64 *output, unsigned int len, - u64 *iv); - -extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input, - u64 *output, unsigned int len, - u64 *iv); - -extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input, - u64 *output, unsigned int len, - u64 *iv); - -extern void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, - u64 *output, unsigned int len, - u64 *iv); -extern void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input, - u64 *output, unsigned int len, - u64 *iv); -extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input, - u64 *output, unsigned int len, - u64 *iv); - static struct aes_ops aes128_ops = { - .encrypt = aes_sparc64_encrypt_128, - .decrypt = aes_sparc64_decrypt_128, .load_encrypt_keys = aes_sparc64_load_encrypt_keys_128, .load_decrypt_keys = aes_sparc64_load_decrypt_keys_128, .ecb_encrypt = aes_sparc64_ecb_encrypt_128, @@ -138,8 +64,6 @@ static struct aes_ops aes128_ops = { }; static struct aes_ops aes192_ops = { - .encrypt = aes_sparc64_encrypt_192, - .decrypt = aes_sparc64_decrypt_192, .load_encrypt_keys = aes_sparc64_load_encrypt_keys_192, .load_decrypt_keys = aes_sparc64_load_decrypt_keys_192, .ecb_encrypt = aes_sparc64_ecb_encrypt_192, @@ -150,8 +74,6 @@ static struct aes_ops aes192_ops = { }; static struct aes_ops aes256_ops = { - .encrypt = aes_sparc64_encrypt_256, - .decrypt = aes_sparc64_decrypt_256, .load_encrypt_keys = aes_sparc64_load_encrypt_keys_256, .load_decrypt_keys = aes_sparc64_load_decrypt_keys_256, .ecb_encrypt = aes_sparc64_ecb_encrypt_256, @@ -161,13 +83,10 @@ static struct aes_ops aes256_ops = { .ctr_crypt = aes_sparc64_ctr_crypt_256, }; -extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key, - unsigned int key_len); - -static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, - unsigned int key_len) +static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) { - struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm); switch (key_len) { case AES_KEYSIZE_128: @@ -195,26 +114,6 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, return 0; } -static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key, - unsigned int key_len) -{ - return aes_set_key(crypto_skcipher_tfm(tfm), in_key, key_len); -} - -static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst); -} - -static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst); -} - static int ecb_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -358,26 +257,6 @@ static int ctr_crypt(struct skcipher_request *req) return err; } -static struct crypto_alg cipher_alg = { - .cra_name = "aes", - .cra_driver_name = "aes-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), - .cra_alignmask = 3, - .cra_module = THIS_MODULE, - .cra_u = { - .cipher = { - .cia_min_keysize = AES_MIN_KEY_SIZE, - .cia_max_keysize = AES_MAX_KEY_SIZE, - .cia_setkey = aes_set_key, - .cia_encrypt = crypto_aes_encrypt, - .cia_decrypt = crypto_aes_decrypt - } - } -}; - static struct skcipher_alg skcipher_algs[] = { { .base.cra_name = "ecb(aes)", @@ -440,26 +319,17 @@ static bool __init sparc64_has_aes_opcode(void) static int __init aes_sparc64_mod_init(void) { - int err; - if (!sparc64_has_aes_opcode()) { pr_info("sparc64 aes opcodes not available.\n"); return -ENODEV; } pr_info("Using sparc64 aes opcodes optimized AES implementation\n"); - err = crypto_register_alg(&cipher_alg); - if (err) - return err; - err = crypto_register_skciphers(skcipher_algs, - ARRAY_SIZE(skcipher_algs)); - if (err) - crypto_unregister_alg(&cipher_alg); - return err; + return crypto_register_skciphers(skcipher_algs, + ARRAY_SIZE(skcipher_algs)); } static void __exit aes_sparc64_mod_fini(void) { - crypto_unregister_alg(&cipher_alg); crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); } diff --git a/include/crypto/aes.h b/include/crypto/aes.h index 19fd99f383fb..4a56aed59973 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -49,6 +49,9 @@ union aes_enckey_arch { #elif defined(CONFIG_S390) /* Used when the CPU supports CPACF AES for this key's length */ u8 raw_key[AES_MAX_KEY_SIZE]; +#elif defined(CONFIG_SPARC64) + /* Used when the CPU supports the SPARC64 AES opcodes */ + u64 sparc_rndkeys[AES_MAX_KEYLENGTH / sizeof(u64)]; #endif #endif /* CONFIG_CRYPTO_LIB_AES_ARCH */ }; @@ -199,6 +202,45 @@ void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len, void aes_p8_xts_decrypt(const u8 *in, u8 *out, size_t len, const struct p8_aes_key *key1, const struct p8_aes_key *key2, u8 *iv); +#elif defined(CONFIG_SPARC64) +void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key, + unsigned int key_len); +void aes_sparc64_load_encrypt_keys_128(const u64 *key); +void aes_sparc64_load_encrypt_keys_192(const u64 *key); +void aes_sparc64_load_encrypt_keys_256(const u64 *key); +void aes_sparc64_load_decrypt_keys_128(const u64 *key); +void aes_sparc64_load_decrypt_keys_192(const u64 *key); +void aes_sparc64_load_decrypt_keys_256(const u64 *key); +void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input, u64 *output, + unsigned int len); +void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input, u64 *output, + unsigned int len); +void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input, u64 *output, + unsigned int len); +void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input, u64 *output, + unsigned int len); +void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input, u64 *output, + unsigned int len); +void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input, u64 *output, + unsigned int len); +void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); #endif /** diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 56a9b4f53b0e..920d96e6b498 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -20,6 +20,7 @@ config CRYPTO_LIB_AES_ARCH default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS default y if S390 + default y if SPARC64 config CRYPTO_LIB_AESCFB tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 811b60787dd5..761d52d91f92 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -51,6 +51,7 @@ endif # !CONFIG_SPE endif # CONFIG_PPC libaes-$(CONFIG_RISCV) += riscv/aes-riscv64-zvkned.o +libaes-$(CONFIG_SPARC) += sparc/aes_asm.o endif # CONFIG_CRYPTO_LIB_AES_ARCH ################################################################################ diff --git a/lib/crypto/sparc/aes.h b/lib/crypto/sparc/aes.h new file mode 100644 index 000000000000..e354aa507ee0 --- /dev/null +++ b/lib/crypto/sparc/aes.h @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AES accelerated using the sparc64 aes opcodes + * + * Copyright (C) 2008, Intel Corp. + * Copyright (c) 2010, Intel Corporation. + * Copyright 2026 Google LLC + */ + +#include +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes_opcodes); + +EXPORT_SYMBOL_GPL(aes_sparc64_key_expand); +EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_128); +EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_192); +EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_256); +EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_128); +EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_192); +EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_256); +EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_128); +EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_192); +EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_256); +EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_128); +EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_192); +EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_256); +EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_128); +EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_192); +EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_256); +EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_128); +EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_192); +EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_256); +EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_128); +EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_192); +EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_256); + +void aes_sparc64_encrypt_128(const u64 *key, const u32 *input, u32 *output); +void aes_sparc64_encrypt_192(const u64 *key, const u32 *input, u32 *output); +void aes_sparc64_encrypt_256(const u64 *key, const u32 *input, u32 *output); +void aes_sparc64_decrypt_128(const u64 *key, const u32 *input, u32 *output); +void aes_sparc64_decrypt_192(const u64 *key, const u32 *input, u32 *output); +void aes_sparc64_decrypt_256(const u64 *key, const u32 *input, u32 *output); + +static void aes_preparekey_arch(union aes_enckey_arch *k, + union aes_invkey_arch *inv_k, + const u8 *in_key, int key_len, int nrounds) +{ + if (static_branch_likely(&have_aes_opcodes)) { + u32 aligned_key[AES_MAX_KEY_SIZE / 4]; + + if (IS_ALIGNED((uintptr_t)in_key, 4)) { + aes_sparc64_key_expand((const u32 *)in_key, + k->sparc_rndkeys, key_len); + } else { + memcpy(aligned_key, in_key, key_len); + aes_sparc64_key_expand(aligned_key, + k->sparc_rndkeys, key_len); + memzero_explicit(aligned_key, key_len); + } + /* + * Note that nothing needs to be written to inv_k (if it's + * non-NULL) here, since the SPARC64 assembly code uses + * k->sparc_rndkeys for both encryption and decryption. + */ + } else { + aes_expandkey_generic(k->rndkeys, + inv_k ? inv_k->inv_rndkeys : NULL, + in_key, key_len); + } +} + +static void aes_sparc64_encrypt(const struct aes_enckey *key, + const u32 *input, u32 *output) +{ + if (key->len == AES_KEYSIZE_128) + aes_sparc64_encrypt_128(key->k.sparc_rndkeys, input, output); + else if (key->len == AES_KEYSIZE_192) + aes_sparc64_encrypt_192(key->k.sparc_rndkeys, input, output); + else + aes_sparc64_encrypt_256(key->k.sparc_rndkeys, input, output); +} + +static void aes_encrypt_arch(const struct aes_enckey *key, + u8 out[AES_BLOCK_SIZE], + const u8 in[AES_BLOCK_SIZE]) +{ + u32 bounce_buf[AES_BLOCK_SIZE / 4]; + + if (static_branch_likely(&have_aes_opcodes)) { + if (IS_ALIGNED((uintptr_t)in | (uintptr_t)out, 4)) { + aes_sparc64_encrypt(key, (const u32 *)in, (u32 *)out); + } else { + memcpy(bounce_buf, in, AES_BLOCK_SIZE); + aes_sparc64_encrypt(key, bounce_buf, bounce_buf); + memcpy(out, bounce_buf, AES_BLOCK_SIZE); + } + } else { + aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in); + } +} + +static void aes_sparc64_decrypt(const struct aes_key *key, + const u32 *input, u32 *output) +{ + if (key->len == AES_KEYSIZE_128) + aes_sparc64_decrypt_128(key->k.sparc_rndkeys, input, output); + else if (key->len == AES_KEYSIZE_192) + aes_sparc64_decrypt_192(key->k.sparc_rndkeys, input, output); + else + aes_sparc64_decrypt_256(key->k.sparc_rndkeys, input, output); +} + +static void aes_decrypt_arch(const struct aes_key *key, + u8 out[AES_BLOCK_SIZE], + const u8 in[AES_BLOCK_SIZE]) +{ + u32 bounce_buf[AES_BLOCK_SIZE / 4]; + + if (static_branch_likely(&have_aes_opcodes)) { + if (IS_ALIGNED((uintptr_t)in | (uintptr_t)out, 4)) { + aes_sparc64_decrypt(key, (const u32 *)in, (u32 *)out); + } else { + memcpy(bounce_buf, in, AES_BLOCK_SIZE); + aes_sparc64_decrypt(key, bounce_buf, bounce_buf); + memcpy(out, bounce_buf, AES_BLOCK_SIZE); + } + } else { + aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, + out, in); + } +} + +#define aes_mod_init_arch aes_mod_init_arch +static void aes_mod_init_arch(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_AES)) + return; + + static_branch_enable(&have_aes_opcodes); +} diff --git a/arch/sparc/crypto/aes_asm.S b/lib/crypto/sparc/aes_asm.S similarity index 100% rename from arch/sparc/crypto/aes_asm.S rename to lib/crypto/sparc/aes_asm.S