torvalds/linux
torvalds/linux
2
0
Fork 0
mirror of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-09-04 20:19:47 +08:00
Code

crypto: lib/poly1305 - Use block-only interface

Browse Source 
Now that every architecture provides a block function, use that
to implement the lib/poly1305 and remove the old per-arch code.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Herbert Xu 2025-04-28 12:56:28 +08:00
parent ceef731b0e
commit 10a6d72ea3
7 changed files with 32 additions and 358 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
arch/arm/lib/crypto/poly1305-glue.c
View File

@ -12,7 +12,6 @@
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/unaligned.h>
asmlinkage void poly1305_block_init_arch(
@ -35,17 +34,6 @@ void __weak poly1305_blocks_neon(struct poly1305_block_state *state,
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
{
dctx->s[0] = get_unaligned_le32(key + 16);
dctx->s[1] = get_unaligned_le32(key + 20);
dctx->s[2] = get_unaligned_le32(key + 24);
dctx->s[3] = get_unaligned_le32(key + 28);
dctx->buflen = 0;
poly1305_block_init_arch(&dctx->state, key);
}
EXPORT_SYMBOL(poly1305_init_arch);
void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
unsigned int len, u32 padbit)
{
@ -67,51 +55,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
}
EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
unsigned int nbytes)
{
if (unlikely(dctx->buflen)) {
u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen);
memcpy(dctx->buf + dctx->buflen, src, bytes);
src += bytes;
nbytes -= bytes;
dctx->buflen += bytes;
if (dctx->buflen == POLY1305_BLOCK_SIZE) {
poly1305_blocks_arch(&dctx->state, dctx->buf,
POLY1305_BLOCK_SIZE, 1);
dctx->buflen = 0;
}
}
if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {
poly1305_blocks_arch(&dctx->state, src, nbytes, 1);
src += round_down(nbytes, POLY1305_BLOCK_SIZE);
nbytes %= POLY1305_BLOCK_SIZE;
}
if (unlikely(nbytes)) {
dctx->buflen = nbytes;
memcpy(dctx->buf, src, nbytes);
}
}
EXPORT_SYMBOL(poly1305_update_arch);
void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
{
if (unlikely(dctx->buflen)) {
dctx->buf[dctx->buflen++] = 1;
memset(dctx->buf + dctx->buflen, 0,
POLY1305_BLOCK_SIZE - dctx->buflen);
poly1305_blocks_arch(&dctx->state, dctx->buf, POLY1305_BLOCK_SIZE, 0);
}
poly1305_emit_arch(&dctx->h, dst, dctx->s);
*dctx = (struct poly1305_desc_ctx){};
}
EXPORT_SYMBOL(poly1305_final_arch);
bool poly1305_is_arch_optimized(void)
{
/* We always can use at least the ARM scalar implementation. */

58
arch/arm64/lib/crypto/poly1305-glue.c
View File

@ -12,7 +12,6 @@
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/unaligned.h>
asmlinkage void poly1305_block_init_arch(
@ -30,17 +29,6 @@ EXPORT_SYMBOL_GPL(poly1305_emit_arch);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
{
dctx->s[0] = get_unaligned_le32(key + 16);
dctx->s[1] = get_unaligned_le32(key + 20);
dctx->s[2] = get_unaligned_le32(key + 24);
dctx->s[3] = get_unaligned_le32(key + 28);
dctx->buflen = 0;
poly1305_block_init_arch(&dctx->state, key);
}
EXPORT_SYMBOL(poly1305_init_arch);
void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
unsigned int len, u32 padbit)
{
@ -61,52 +49,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
}
EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
unsigned int nbytes)
{
if (unlikely(dctx->buflen)) {
u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen);
memcpy(dctx->buf + dctx->buflen, src, bytes);
src += bytes;
nbytes -= bytes;
dctx->buflen += bytes;
if (dctx->buflen == POLY1305_BLOCK_SIZE) {
poly1305_blocks_arch(&dctx->state, dctx->buf,
POLY1305_BLOCK_SIZE, 1);
dctx->buflen = 0;
}
}
if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {
poly1305_blocks_arch(&dctx->state, src, nbytes, 1);
src += round_down(nbytes, POLY1305_BLOCK_SIZE);
nbytes %= POLY1305_BLOCK_SIZE;
}
if (unlikely(nbytes)) {
dctx->buflen = nbytes;
memcpy(dctx->buf, src, nbytes);
}
}
EXPORT_SYMBOL(poly1305_update_arch);
void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
{
if (unlikely(dctx->buflen)) {
dctx->buf[dctx->buflen++] = 1;
memset(dctx->buf + dctx->buflen, 0,
POLY1305_BLOCK_SIZE - dctx->buflen);
poly1305_blocks_arch(&dctx->state, dctx->buf,
POLY1305_BLOCK_SIZE, 0);
}
poly1305_emit_arch(&dctx->h, dst, dctx->s);
memzero_explicit(dctx, sizeof(*dctx));
}
EXPORT_SYMBOL(poly1305_final_arch);
bool poly1305_is_arch_optimized(void)
{
/* We always can use at least the ARM64 scalar implementation. */

60
arch/mips/lib/crypto/poly1305-glue.c
View File

@ -9,7 +9,6 @@
#include <linux/cpufeature.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/unaligned.h>
asmlinkage void poly1305_block_init_arch(
@ -24,65 +23,6 @@ asmlinkage void poly1305_emit_arch(const struct poly1305_state *state,
const u32 nonce[4]);
EXPORT_SYMBOL_GPL(poly1305_emit_arch);
void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
{
dctx->s[0] = get_unaligned_le32(key + 16);
dctx->s[1] = get_unaligned_le32(key + 20);
dctx->s[2] = get_unaligned_le32(key + 24);
dctx->s[3] = get_unaligned_le32(key + 28);
dctx->buflen = 0;
poly1305_block_init_arch(&dctx->state, key);
}
EXPORT_SYMBOL(poly1305_init_arch);
void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
unsigned int nbytes)
{
if (unlikely(dctx->buflen)) {
u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen);
memcpy(dctx->buf + dctx->buflen, src, bytes);
src += bytes;
nbytes -= bytes;
dctx->buflen += bytes;
if (dctx->buflen == POLY1305_BLOCK_SIZE) {
poly1305_blocks_arch(&dctx->state, dctx->buf,
POLY1305_BLOCK_SIZE, 1);
dctx->buflen = 0;
}
}
if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {
unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE);
poly1305_blocks_arch(&dctx->state, src, len, 1);
src += len;
nbytes %= POLY1305_BLOCK_SIZE;
}
if (unlikely(nbytes)) {
dctx->buflen = nbytes;
memcpy(dctx->buf, src, nbytes);
}
}
EXPORT_SYMBOL(poly1305_update_arch);
void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
{
if (unlikely(dctx->buflen)) {
dctx->buf[dctx->buflen++] = 1;
memset(dctx->buf + dctx->buflen, 0,
POLY1305_BLOCK_SIZE - dctx->buflen);
poly1305_blocks_arch(&dctx->state, dctx->buf,
POLY1305_BLOCK_SIZE, 0);
}
poly1305_emit_arch(&dctx->h, dst, dctx->s);
*dctx = (struct poly1305_desc_ctx){};
}
EXPORT_SYMBOL(poly1305_final_arch);
bool poly1305_is_arch_optimized(void)
{
return true;

63
arch/powerpc/lib/crypto/poly1305-p10-glue.c
View File

@ -10,7 +10,6 @@
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/unaligned.h>
asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen);
@ -45,17 +44,6 @@ void poly1305_block_init_arch(struct poly1305_block_state *dctx,
}
EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
{
dctx->s[0] = get_unaligned_le32(key + 16);
dctx->s[1] = get_unaligned_le32(key + 20);
dctx->s[2] = get_unaligned_le32(key + 24);
dctx->s[3] = get_unaligned_le32(key + 28);
dctx->buflen = 0;
poly1305_block_init_arch(&dctx->state, key);
}
EXPORT_SYMBOL(poly1305_init_arch);
void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
unsigned int len, u32 padbit)
{
@ -76,57 +64,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
}
EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
void poly1305_update_arch(struct poly1305_desc_ctx *dctx,
const u8 *src, unsigned int srclen)
{
unsigned int bytes;
if (!static_key_enabled(&have_p10))
return poly1305_update_generic(dctx, src, srclen);
if (unlikely(dctx->buflen)) {
bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen);
memcpy(dctx->buf + dctx->buflen, src, bytes);
src += bytes;
srclen -= bytes;
dctx->buflen += bytes;
if (dctx->buflen < POLY1305_BLOCK_SIZE)
return;
poly1305_blocks_arch(&dctx->state, dctx->buf,
POLY1305_BLOCK_SIZE, 1);
dctx->buflen = 0;
}
if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
poly1305_blocks_arch(&dctx->state, src, srclen, 1);
src += srclen - (srclen % POLY1305_BLOCK_SIZE);
srclen %= POLY1305_BLOCK_SIZE;
}
if (unlikely(srclen)) {
dctx->buflen = srclen;
memcpy(dctx->buf, src, srclen);
}
}
EXPORT_SYMBOL(poly1305_update_arch);
void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
{
if (!static_key_enabled(&have_p10))
return poly1305_final_generic(dctx, dst);
if (dctx->buflen) {
dctx->buf[dctx->buflen++] = 1;
memset(dctx->buf + dctx->buflen, 0,
POLY1305_BLOCK_SIZE - dctx->buflen);
poly1305_blocks_arch(&dctx->state, dctx->buf,
POLY1305_BLOCK_SIZE, 0);
}
poly1305_emit_arch(&dctx->h, dst, dctx->s);
}
EXPORT_SYMBOL(poly1305_final_arch);
bool poly1305_is_arch_optimized(void)
{
return static_key_enabled(&have_p10);

60
arch/x86/lib/crypto/poly1305_glue.c
View File

@ -10,7 +10,6 @@
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sizes.h>
#include <linux/string.h>
#include <linux/unaligned.h>
struct poly1305_arch_internal {
@ -96,65 +95,6 @@ void poly1305_emit_arch(const struct poly1305_state *ctx,
}
EXPORT_SYMBOL_GPL(poly1305_emit_arch);
void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
{
dctx->s[0] = get_unaligned_le32(&key[16]);
dctx->s[1] = get_unaligned_le32(&key[20]);
dctx->s[2] = get_unaligned_le32(&key[24]);
dctx->s[3] = get_unaligned_le32(&key[28]);
dctx->buflen = 0;
poly1305_block_init_arch(&dctx->state, key);
}
EXPORT_SYMBOL(poly1305_init_arch);
void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
unsigned int srclen)
{
unsigned int bytes;
if (unlikely(dctx->buflen)) {
bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen);
memcpy(dctx->buf + dctx->buflen, src, bytes);
src += bytes;
srclen -= bytes;
dctx->buflen += bytes;
if (dctx->buflen == POLY1305_BLOCK_SIZE) {
poly1305_blocks_arch(&dctx->state, dctx->buf,
POLY1305_BLOCK_SIZE, 1);
dctx->buflen = 0;
}
}
if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
bytes = round_down(srclen, POLY1305_BLOCK_SIZE);
poly1305_blocks_arch(&dctx->state, src, bytes, 1);
src += bytes;
srclen -= bytes;
}
if (unlikely(srclen)) {
dctx->buflen = srclen;
memcpy(dctx->buf, src, srclen);
}
}
EXPORT_SYMBOL(poly1305_update_arch);
void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
{
if (unlikely(dctx->buflen)) {
dctx->buf[dctx->buflen++] = 1;
memset(dctx->buf + dctx->buflen, 0,
POLY1305_BLOCK_SIZE - dctx->buflen);
poly1305_blocks_arch(&dctx->state, dctx->buf,
POLY1305_BLOCK_SIZE, 0);
}
poly1305_emit_arch(&dctx->h, dst, dctx->s);
memzero_explicit(dctx, sizeof(*dctx));
}
EXPORT_SYMBOL(poly1305_final_arch);
bool poly1305_is_arch_optimized(void)
{
return static_key_enabled(&poly1305_use_avx);

53
include/crypto/poly1305.h
View File

@ -55,55 +55,14 @@ struct poly1305_desc_ctx {
unsigned int buflen;
/* finalize key */
u32 s[4];
union {
struct {
struct poly1305_state h;
union {
struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE];
struct poly1305_core_key core_r;
};
};
struct poly1305_block_state state;
};
struct poly1305_block_state state;
};
void poly1305_init_arch(struct poly1305_desc_ctx *desc,
const u8 key[POLY1305_KEY_SIZE]);
void poly1305_init_generic(struct poly1305_desc_ctx *desc,
const u8 key[POLY1305_KEY_SIZE]);
static inline void poly1305_init(struct poly1305_desc_ctx *desc, const u8 *key)
{
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
poly1305_init_arch(desc, key);
else
poly1305_init_generic(desc, key);
}
void poly1305_update_arch(struct poly1305_desc_ctx *desc, const u8 *src,
unsigned int nbytes);
void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src,
unsigned int nbytes);
static inline void poly1305_update(struct poly1305_desc_ctx *desc,
const u8 *src, unsigned int nbytes)
{
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
poly1305_update_arch(desc, src, nbytes);
else
poly1305_update_generic(desc, src, nbytes);
}
void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *digest);
void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *digest);
static inline void poly1305_final(struct poly1305_desc_ctx *desc, u8 *digest)
{
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
poly1305_final_arch(desc, digest);
else
poly1305_final_generic(desc, digest);
}
void poly1305_init(struct poly1305_desc_ctx *desc,
const u8 key[POLY1305_KEY_SIZE]);
void poly1305_update(struct poly1305_desc_ctx *desc,
const u8 *src, unsigned int nbytes);
void poly1305_final(struct poly1305_desc_ctx *desc, u8 *digest);
#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)
bool poly1305_is_arch_optimized(void);

39
lib/crypto/poly1305.c
View File

@ -22,47 +22,60 @@ void poly1305_block_init_generic(struct poly1305_block_state *desc,
}
EXPORT_SYMBOL_GPL(poly1305_block_init_generic);
void poly1305_init_generic(struct poly1305_desc_ctx *desc,
const u8 key[POLY1305_KEY_SIZE])
void poly1305_init(struct poly1305_desc_ctx *desc,
const u8 key[POLY1305_KEY_SIZE])
{
desc->s[0] = get_unaligned_le32(key + 16);
desc->s[1] = get_unaligned_le32(key + 20);
desc->s[2] = get_unaligned_le32(key + 24);
desc->s[3] = get_unaligned_le32(key + 28);
desc->buflen = 0;
poly1305_block_init_generic(&desc->state, key);
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
poly1305_block_init_arch(&desc->state, key);
else
poly1305_block_init_generic(&desc->state, key);
}
EXPORT_SYMBOL_GPL(poly1305_init_generic);
EXPORT_SYMBOL(poly1305_init);
static inline void poly1305_blocks(struct poly1305_block_state *state,
const u8 *src, unsigned int len)
{
poly1305_blocks_generic(state, src, len, 1);
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
poly1305_blocks_arch(state, src, len, 1);
else
poly1305_blocks_generic(state, src, len, 1);
}
void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src,
unsigned int nbytes)
void poly1305_update(struct poly1305_desc_ctx *desc,
const u8 *src, unsigned int nbytes)
{
desc->buflen = BLOCK_HASH_UPDATE(poly1305_blocks, &desc->state,
src, nbytes, POLY1305_BLOCK_SIZE,
desc->buf, desc->buflen);
}
EXPORT_SYMBOL_GPL(poly1305_update_generic);
EXPORT_SYMBOL(poly1305_update);
void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *dst)
void poly1305_final(struct poly1305_desc_ctx *desc, u8 *dst)
{
if (unlikely(desc->buflen)) {
desc->buf[desc->buflen++] = 1;
memset(desc->buf + desc->buflen, 0,
POLY1305_BLOCK_SIZE - desc->buflen);
poly1305_blocks_generic(&desc->state, desc->buf,
POLY1305_BLOCK_SIZE, 0);
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
poly1305_blocks_arch(&desc->state, desc->buf,
POLY1305_BLOCK_SIZE, 0);
else
poly1305_blocks_generic(&desc->state, desc->buf,
POLY1305_BLOCK_SIZE, 0);
}
poly1305_emit_generic(&desc->h, dst, desc->s);
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
poly1305_emit_arch(&desc->state.h, dst, desc->s);
else
poly1305_emit_generic(&desc->state.h, dst, desc->s);
*desc = (struct poly1305_desc_ctx){};
}
EXPORT_SYMBOL_GPL(poly1305_final_generic);
EXPORT_SYMBOL(poly1305_final);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");