MIRRORS/linux-imx
1
0
Fork 0
mirror of https://github.com/nxp-imx/linux-imx.git synced 2025-07-06 17:35:20 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

BACKPORT: FROMLIST: dm-verity: improve performance by using multibuffer hashing

Browse Source 
When supported by the hash algorithm, use crypto_shash_finup_mb() to
interleave the hashing of pairs of data blocks.  On some CPUs this
nearly doubles hashing performance.  The increase in overall throughput
of cold-cache dm-verity reads that I'm seeing on arm64 and x86_64 is
roughly 35% (though this metric is hard to measure as it jumps around a
lot).

For now this is only done on data blocks, not Merkle tree blocks.  We
could use finup_mb on Merkle tree blocks too, but that is less important
as there aren't as many Merkle tree blocks as data blocks, and that
would require some additional code restructuring.

Reviewed-by: Sami Tolvanen <samitolvanen@google.com>
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Eric Biggers <ebiggers@google.com>

Bug: 330611177
Link: https://lore.kernel.org/r/20240621165922.77672-16-ebiggers@kernel.org
(resolved conflict due to missing the upstream commit
 "dm-verity: Convert from tasklet to BH workqueue", which is hard to
 cherry-pick because it depends on workqueue subsystem changes)
Change-Id: I5a2ee7af05b53e30a3bc8a1e1ffc77a5244cb38d
Signed-off-by: Eric Biggers <ebiggers@google.com>
This commit is contained in:
Eric Biggers 2024-06-21 10:38:00 -07:00 committed by William McVicker
parent 6c33cbb433
commit da5b43867d
2 changed files with 147 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

166
drivers/md/dm-verity-target.c
View File

@ -183,18 +183,24 @@ out:
return r;
}
static int verity_ahash(struct dm_verity *v, struct dm_verity_io *io,
const u8 *data, size_t len, u8 *digest, bool may_sleep)
{
struct ahash_request *req = verity_io_hash_req(v, io);
struct crypto_wait wait;
return verity_ahash_init(v, req, &wait, may_sleep) ?:
verity_ahash_update(v, req, data, len, &wait) ?:
verity_ahash_final(v, req, digest, &wait);
}
int verity_hash(struct dm_verity *v, struct dm_verity_io *io,
const u8 *data, size_t len, u8 *digest, bool may_sleep)
{
int r;
if (static_branch_unlikely(&ahash_enabled) && !v->shash_tfm) {
struct ahash_request *req = verity_io_hash_req(v, io);
struct crypto_wait wait;
r = verity_ahash_init(v, req, &wait, may_sleep) ?:
verity_ahash_update(v, req, data, len, &wait) ?:
verity_ahash_final(v, req, digest, &wait);
r = verity_ahash(v, io, data, len, digest, may_sleep);
} else {
struct shash_desc *desc = verity_io_hash_req(v, io);
@ -207,6 +213,34 @@ int verity_hash(struct dm_verity *v, struct dm_verity_io *io,
return r;
}
static int verity_hash_mb(struct dm_verity *v, struct dm_verity_io *io,
const u8 *data[], size_t len, u8 *digests[],
int num_blocks)
{
int r = 0;
if (static_branch_unlikely(&ahash_enabled) && !v->shash_tfm) {
int i;
/* Note: in practice num_blocks is always 1 in this case. */
for (i = 0; i < num_blocks; i++) {
r = verity_ahash(v, io, data[i], len, digests[i],
!io->in_tasklet);
if (r)
break;
}
} else {
struct shash_desc *desc = verity_io_hash_req(v, io);
desc->tfm = v->shash_tfm;
r = crypto_shash_import(desc, v->initial_hashstate) ?:
crypto_shash_finup_mb(desc, data, len, digests, num_blocks);
}
if (unlikely(r))
DMERR("Error hashing blocks: %d", r);
return r;
}
static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
sector_t *hash_block, unsigned int *offset)
{
@ -457,9 +491,12 @@ free_ret:
static int verity_handle_data_hash_mismatch(struct dm_verity *v,
struct dm_verity_io *io,
struct bio *bio,
const u8 *want_digest,
sector_t blkno, u8 *data)
struct pending_block *block)
{
const u8 *want_digest = block->want_digest;
sector_t blkno = block->blkno;
u8 *data = block->data;
if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) {
/*
* Error handling code (FEC included) cannot be run in a
@ -487,6 +524,53 @@ static int verity_handle_data_hash_mismatch(struct dm_verity *v,
return 0;
}
static void verity_clear_pending_blocks(struct dm_verity_io *io)
{
int i;
for (i = io->num_pending - 1; i >= 0; i--) {
kunmap_local(io->pending_blocks[i].data);
io->pending_blocks[i].data = NULL;
}
io->num_pending = 0;
}
static int verity_verify_pending_blocks(struct dm_verity *v,
struct dm_verity_io *io,
struct bio *bio)
{
const u8 *data[DM_VERITY_MAX_PENDING_DATA_BLOCKS];
u8 *real_digests[DM_VERITY_MAX_PENDING_DATA_BLOCKS];
int i;
int r;
for (i = 0; i < io->num_pending; i++) {
data[i] = io->pending_blocks[i].data;
real_digests[i] = io->pending_blocks[i].real_digest;
}
r = verity_hash_mb(v, io, data, 1 << v->data_dev_block_bits,
real_digests, io->num_pending);
if (unlikely(r))
return r;
for (i = 0; i < io->num_pending; i++) {
struct pending_block *block = &io->pending_blocks[i];
if (likely(memcmp(block->real_digest, block->want_digest,
v->digest_size) == 0)) {
if (v->validated_blocks)
set_bit(block->blkno, v->validated_blocks);
} else {
r = verity_handle_data_hash_mismatch(v, io, bio, block);
if (unlikely(r))
return r;
}
}
verity_clear_pending_blocks(io);
return 0;
}
/*
* Verify one "dm_verity_io" structure.
*/
@ -498,6 +582,9 @@ static int verity_verify_io(struct dm_verity_io *io)
struct bvec_iter *iter;
struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
unsigned int b;
int r;
io->num_pending = 0;
if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) {
/*
@ -511,21 +598,22 @@ static int verity_verify_io(struct dm_verity_io *io)
for (b = 0; b < io->n_blocks;
b++, bio_advance_iter(bio, iter, block_size)) {
int r;
sector_t cur_block = io->block + b;
sector_t blkno = io->block + b;
struct pending_block *block;
bool is_zero;
struct bio_vec bv;
void *data;
if (v->validated_blocks && bio->bi_status == BLK_STS_OK &&
likely(test_bit(cur_block, v->validated_blocks)))
likely(test_bit(blkno, v->validated_blocks)))
continue;
r = verity_hash_for_block(v, io, cur_block,
verity_io_want_digest(v, io),
block = &io->pending_blocks[io->num_pending];
r = verity_hash_for_block(v, io, blkno, block->want_digest,
&is_zero);
if (unlikely(r < 0))
return r;
goto error;
bv = bio_iter_iovec(bio, *iter);
if (unlikely(bv.bv_len < block_size)) {
@ -536,7 +624,8 @@ static int verity_verify_io(struct dm_verity_io *io)
* data block size to be greater than PAGE_SIZE.
*/
DMERR_LIMIT("unaligned io (data block spans pages)");
return -EIO;
r = -EIO;
goto error;
}
data = bvec_kmap_local(&bv);
@ -550,30 +639,26 @@ static int verity_verify_io(struct dm_verity_io *io)
kunmap_local(data);
continue;
}
r = verity_hash(v, io, data, block_size,
verity_io_real_digest(v, io), !io->in_tasklet);
if (unlikely(r < 0)) {
kunmap_local(data);
return r;
block->data = data;
block->blkno = blkno;
if (++io->num_pending == v->mb_max_msgs) {
r = verity_verify_pending_blocks(v, io, bio);
if (unlikely(r))
goto error;
}
}
if (likely(memcmp(verity_io_real_digest(v, io),
verity_io_want_digest(v, io), v->digest_size) == 0)) {
if (v->validated_blocks)
set_bit(cur_block, v->validated_blocks);
kunmap_local(data);
continue;
}
r = verity_handle_data_hash_mismatch(v, io, bio,
verity_io_want_digest(v, io),
cur_block, data);
kunmap_local(data);
if (io->num_pending) {
r = verity_verify_pending_blocks(v, io, bio);
if (unlikely(r))
return r;
goto error;
}
return 0;
error:
verity_clear_pending_blocks(io);
return r;
}
/*
@ -1136,10 +1221,11 @@ static int verity_setup_hash_alg(struct dm_verity *v, const char *alg_name)
* Allocate the hash transformation object that this dm-verity instance
* will use. The vast majority of dm-verity users use CPU-based
* hashing, so when possible use the shash API to minimize the crypto
* API overhead. If the ahash API resolves to a different driver
* (likely an off-CPU hardware offload), use ahash instead. Also use
* ahash if the obsolete dm-verity format with the appended salt is
* being used, so that quirk only needs to be handled in one place.
* API overhead, especially when multibuffer hashing is used. If the
* ahash API resolves to a different driver (likely an off-CPU hardware
* offload), use ahash instead. Also use ahash if the obsolete
* dm-verity format with the appended salt is being used, so that quirk
* only needs to be handled in one place.
*/
ahash = crypto_alloc_ahash(alg_name, 0,
v->use_tasklet ? CRYPTO_ALG_ASYNC : 0);
@ -1167,13 +1253,17 @@ static int verity_setup_hash_alg(struct dm_verity *v, const char *alg_name)
v->digest_size = crypto_shash_digestsize(shash);
v->hash_reqsize = sizeof(struct shash_desc) +
crypto_shash_descsize(shash);
DMINFO("%s using shash \"%s\"", alg_name, driver_name);
v->mb_max_msgs = min(crypto_shash_mb_max_msgs(shash),
DM_VERITY_MAX_PENDING_DATA_BLOCKS);
DMINFO("%s using shash \"%s\"%s", alg_name, driver_name,
v->mb_max_msgs > 1 ? " (multibuffer)" : "");
} else {
v->ahash_tfm = ahash;
static_branch_inc(&ahash_enabled);
v->digest_size = crypto_ahash_digestsize(ahash);
v->hash_reqsize = sizeof(struct ahash_request) +
crypto_ahash_reqsize(ahash);
v->mb_max_msgs = 1;
DMINFO("%s using ahash \"%s\"", alg_name, driver_name);
}
if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {

33
drivers/md/dm-verity.h
View File

@ -57,6 +57,7 @@ struct dm_verity {
unsigned char version;
bool hash_failed:1; /* set if hash of any block failed */
bool use_tasklet:1; /* try to verify in tasklet before work-queue */
unsigned char mb_max_msgs; /* max multibuffer hashing interleaving factor */
unsigned int digest_size; /* digest size for the current hash algorithm */
unsigned int hash_reqsize; /* the size of temporary space for crypto */
enum verity_mode mode; /* mode for handling verification errors */
@ -76,6 +77,15 @@ struct dm_verity {
mempool_t recheck_pool;
};
#define DM_VERITY_MAX_PENDING_DATA_BLOCKS HASH_MAX_MB_MSGS
struct pending_block {
void *data;
sector_t blkno;
u8 want_digest[HASH_MAX_DIGESTSIZE];
u8 real_digest[HASH_MAX_DIGESTSIZE];
};
struct dm_verity_io {
struct dm_verity *v;
@ -91,8 +101,15 @@ struct dm_verity_io {
struct work_struct work;
u8 tmp_digest[HASH_MAX_DIGESTSIZE];
u8 real_digest[HASH_MAX_DIGESTSIZE];
u8 want_digest[HASH_MAX_DIGESTSIZE];
/*
* This is the queue of data blocks that are pending verification. We
* allow multiple blocks to be queued up in order to support multibuffer
* hashing, i.e. interleaving the hashing of multiple messages. On many
* CPUs this improves performance significantly.
*/
int num_pending;
struct pending_block pending_blocks[DM_VERITY_MAX_PENDING_DATA_BLOCKS];
/*
* This struct is followed by a variable-sized hash request of size
@ -108,18 +125,6 @@ static inline void *verity_io_hash_req(struct dm_verity *v,
return io + 1;
}
static inline u8 *verity_io_real_digest(struct dm_verity *v,
struct dm_verity_io *io)
{
return io->real_digest;
}
static inline u8 *verity_io_want_digest(struct dm_verity *v,
struct dm_verity_io *io)
{
return io->want_digest;
}
extern int verity_hash(struct dm_verity *v, struct dm_verity_io *io,
const u8 *data, size_t len, u8 *digest, bool may_sleep);