|
|
|
|
@@ -9,14 +9,6 @@
|
|
|
|
|
|
|
|
|
|
#define LZ4_MAX_DISTANCE_PAGES (DIV_ROUND_UP(LZ4_DISTANCE_MAX, PAGE_SIZE) + 1)
|
|
|
|
|
|
|
|
|
|
struct z_erofs_lz4_decompress_ctx {
|
|
|
|
|
struct z_erofs_decompress_req *rq;
|
|
|
|
|
/* # of encoded, decoded pages */
|
|
|
|
|
unsigned int inpages, outpages;
|
|
|
|
|
/* decoded block total length (used for in-place decompression) */
|
|
|
|
|
unsigned int oend;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static int z_erofs_load_lz4_config(struct super_block *sb,
|
|
|
|
|
struct erofs_super_block *dsb, void *data, int size)
|
|
|
|
|
{
|
|
|
|
|
@@ -55,10 +47,9 @@ static int z_erofs_load_lz4_config(struct super_block *sb,
|
|
|
|
|
* Fill all gaps with bounce pages if it's a sparse page list. Also check if
|
|
|
|
|
* all physical pages are consecutive, which can be seen for moderate CR.
|
|
|
|
|
*/
|
|
|
|
|
static int z_erofs_lz4_prepare_dstpages(struct z_erofs_lz4_decompress_ctx *ctx,
|
|
|
|
|
static int z_erofs_lz4_prepare_dstpages(struct z_erofs_decompress_req *rq,
|
|
|
|
|
struct page **pagepool)
|
|
|
|
|
{
|
|
|
|
|
struct z_erofs_decompress_req *rq = ctx->rq;
|
|
|
|
|
struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL };
|
|
|
|
|
unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES,
|
|
|
|
|
BITS_PER_LONG)] = { 0 };
|
|
|
|
|
@@ -68,7 +59,7 @@ static int z_erofs_lz4_prepare_dstpages(struct z_erofs_lz4_decompress_ctx *ctx,
|
|
|
|
|
unsigned int i, j, top;
|
|
|
|
|
|
|
|
|
|
top = 0;
|
|
|
|
|
for (i = j = 0; i < ctx->outpages; ++i, ++j) {
|
|
|
|
|
for (i = j = 0; i < rq->outpages; ++i, ++j) {
|
|
|
|
|
struct page *const page = rq->out[i];
|
|
|
|
|
struct page *victim;
|
|
|
|
|
|
|
|
|
|
@@ -114,36 +105,36 @@ static int z_erofs_lz4_prepare_dstpages(struct z_erofs_lz4_decompress_ctx *ctx,
|
|
|
|
|
return kaddr ? 1 : 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void *z_erofs_lz4_handle_overlap(struct z_erofs_lz4_decompress_ctx *ctx,
|
|
|
|
|
static void *z_erofs_lz4_handle_overlap(struct z_erofs_decompress_req *rq,
|
|
|
|
|
void *inpage, void *out, unsigned int *inputmargin,
|
|
|
|
|
int *maptype, bool may_inplace)
|
|
|
|
|
{
|
|
|
|
|
struct z_erofs_decompress_req *rq = ctx->rq;
|
|
|
|
|
unsigned int omargin, total, i;
|
|
|
|
|
unsigned int oend, omargin, total, i;
|
|
|
|
|
struct page **in;
|
|
|
|
|
void *src, *tmp;
|
|
|
|
|
|
|
|
|
|
if (rq->inplace_io) {
|
|
|
|
|
omargin = PAGE_ALIGN(ctx->oend) - ctx->oend;
|
|
|
|
|
oend = rq->pageofs_out + rq->outputsize;
|
|
|
|
|
omargin = PAGE_ALIGN(oend) - oend;
|
|
|
|
|
if (rq->partial_decoding || !may_inplace ||
|
|
|
|
|
omargin < LZ4_DECOMPRESS_INPLACE_MARGIN(rq->inputsize))
|
|
|
|
|
goto docopy;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < ctx->inpages; ++i)
|
|
|
|
|
if (rq->out[ctx->outpages - ctx->inpages + i] !=
|
|
|
|
|
for (i = 0; i < rq->inpages; ++i)
|
|
|
|
|
if (rq->out[rq->outpages - rq->inpages + i] !=
|
|
|
|
|
rq->in[i])
|
|
|
|
|
goto docopy;
|
|
|
|
|
kunmap_local(inpage);
|
|
|
|
|
*maptype = 3;
|
|
|
|
|
return out + ((ctx->outpages - ctx->inpages) << PAGE_SHIFT);
|
|
|
|
|
return out + ((rq->outpages - rq->inpages) << PAGE_SHIFT);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (ctx->inpages <= 1) {
|
|
|
|
|
if (rq->inpages <= 1) {
|
|
|
|
|
*maptype = 0;
|
|
|
|
|
return inpage;
|
|
|
|
|
}
|
|
|
|
|
kunmap_local(inpage);
|
|
|
|
|
src = erofs_vm_map_ram(rq->in, ctx->inpages);
|
|
|
|
|
src = erofs_vm_map_ram(rq->in, rq->inpages);
|
|
|
|
|
if (!src)
|
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
|
*maptype = 1;
|
|
|
|
|
@@ -152,7 +143,7 @@ static void *z_erofs_lz4_handle_overlap(struct z_erofs_lz4_decompress_ctx *ctx,
|
|
|
|
|
docopy:
|
|
|
|
|
/* Or copy compressed data which can be overlapped to per-CPU buffer */
|
|
|
|
|
in = rq->in;
|
|
|
|
|
src = z_erofs_get_gbuf(ctx->inpages);
|
|
|
|
|
src = z_erofs_get_gbuf(rq->inpages);
|
|
|
|
|
if (!src) {
|
|
|
|
|
DBG_BUGON(1);
|
|
|
|
|
kunmap_local(inpage);
|
|
|
|
|
@@ -197,10 +188,8 @@ int z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, const char *padbuf,
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int z_erofs_lz4_decompress_mem(struct z_erofs_lz4_decompress_ctx *ctx,
|
|
|
|
|
u8 *dst)
|
|
|
|
|
static int z_erofs_lz4_decompress_mem(struct z_erofs_decompress_req *rq, u8 *dst)
|
|
|
|
|
{
|
|
|
|
|
struct z_erofs_decompress_req *rq = ctx->rq;
|
|
|
|
|
bool support_0padding = false, may_inplace = false;
|
|
|
|
|
unsigned int inputmargin;
|
|
|
|
|
u8 *out, *headpage, *src;
|
|
|
|
|
@@ -224,7 +213,7 @@ static int z_erofs_lz4_decompress_mem(struct z_erofs_lz4_decompress_ctx *ctx,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
inputmargin = rq->pageofs_in;
|
|
|
|
|
src = z_erofs_lz4_handle_overlap(ctx, headpage, dst, &inputmargin,
|
|
|
|
|
src = z_erofs_lz4_handle_overlap(rq, headpage, dst, &inputmargin,
|
|
|
|
|
&maptype, may_inplace);
|
|
|
|
|
if (IS_ERR(src))
|
|
|
|
|
return PTR_ERR(src);
|
|
|
|
|
@@ -251,7 +240,7 @@ static int z_erofs_lz4_decompress_mem(struct z_erofs_lz4_decompress_ctx *ctx,
|
|
|
|
|
if (maptype == 0) {
|
|
|
|
|
kunmap_local(headpage);
|
|
|
|
|
} else if (maptype == 1) {
|
|
|
|
|
vm_unmap_ram(src, ctx->inpages);
|
|
|
|
|
vm_unmap_ram(src, rq->inpages);
|
|
|
|
|
} else if (maptype == 2) {
|
|
|
|
|
z_erofs_put_gbuf(src);
|
|
|
|
|
} else if (maptype != 3) {
|
|
|
|
|
@@ -264,54 +253,42 @@ static int z_erofs_lz4_decompress_mem(struct z_erofs_lz4_decompress_ctx *ctx,
|
|
|
|
|
static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq,
|
|
|
|
|
struct page **pagepool)
|
|
|
|
|
{
|
|
|
|
|
struct z_erofs_lz4_decompress_ctx ctx;
|
|
|
|
|
unsigned int dst_maptype;
|
|
|
|
|
void *dst;
|
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
|
|
ctx.rq = rq;
|
|
|
|
|
ctx.oend = rq->pageofs_out + rq->outputsize;
|
|
|
|
|
ctx.outpages = PAGE_ALIGN(ctx.oend) >> PAGE_SHIFT;
|
|
|
|
|
ctx.inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
|
|
|
|
|
|
|
|
|
|
/* one optimized fast path only for non bigpcluster cases yet */
|
|
|
|
|
if (ctx.inpages == 1 && ctx.outpages == 1 && !rq->inplace_io) {
|
|
|
|
|
if (rq->inpages == 1 && rq->outpages == 1 && !rq->inplace_io) {
|
|
|
|
|
DBG_BUGON(!*rq->out);
|
|
|
|
|
dst = kmap_local_page(*rq->out);
|
|
|
|
|
dst_maptype = 0;
|
|
|
|
|
goto dstmap_out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* general decoding path which can be used for all cases */
|
|
|
|
|
ret = z_erofs_lz4_prepare_dstpages(&ctx, pagepool);
|
|
|
|
|
if (ret < 0) {
|
|
|
|
|
return ret;
|
|
|
|
|
} else if (ret > 0) {
|
|
|
|
|
dst = page_address(*rq->out);
|
|
|
|
|
dst_maptype = 1;
|
|
|
|
|
} else {
|
|
|
|
|
dst = erofs_vm_map_ram(rq->out, ctx.outpages);
|
|
|
|
|
if (!dst)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
dst_maptype = 2;
|
|
|
|
|
/* general decoding path which can be used for all cases */
|
|
|
|
|
ret = z_erofs_lz4_prepare_dstpages(rq, pagepool);
|
|
|
|
|
if (ret < 0)
|
|
|
|
|
return ret;
|
|
|
|
|
if (ret > 0) {
|
|
|
|
|
dst = page_address(*rq->out);
|
|
|
|
|
dst_maptype = 1;
|
|
|
|
|
} else {
|
|
|
|
|
dst = erofs_vm_map_ram(rq->out, rq->outpages);
|
|
|
|
|
if (!dst)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
dst_maptype = 2;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
dstmap_out:
|
|
|
|
|
ret = z_erofs_lz4_decompress_mem(&ctx, dst);
|
|
|
|
|
ret = z_erofs_lz4_decompress_mem(rq, dst);
|
|
|
|
|
if (!dst_maptype)
|
|
|
|
|
kunmap_local(dst);
|
|
|
|
|
else if (dst_maptype == 2)
|
|
|
|
|
vm_unmap_ram(dst, ctx.outpages);
|
|
|
|
|
vm_unmap_ram(dst, rq->outpages);
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int z_erofs_transform_plain(struct z_erofs_decompress_req *rq,
|
|
|
|
|
struct page **pagepool)
|
|
|
|
|
{
|
|
|
|
|
const unsigned int nrpages_in =
|
|
|
|
|
PAGE_ALIGN(rq->pageofs_in + rq->inputsize) >> PAGE_SHIFT;
|
|
|
|
|
const unsigned int nrpages_out =
|
|
|
|
|
PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
|
|
|
|
|
const unsigned int nrpages_in = rq->inpages, nrpages_out = rq->outpages;
|
|
|
|
|
const unsigned int bs = rq->sb->s_blocksize;
|
|
|
|
|
unsigned int cur = 0, ni = 0, no, pi, po, insz, cnt;
|
|
|
|
|
u8 *kin;
|
|
|
|
|
@@ -373,7 +350,7 @@ int z_erofs_stream_switch_bufs(struct z_erofs_stream_dctx *dctx, void **dst,
|
|
|
|
|
unsigned int j;
|
|
|
|
|
|
|
|
|
|
if (!dctx->avail_out) {
|
|
|
|
|
if (++dctx->no >= dctx->outpages || !rq->outputsize) {
|
|
|
|
|
if (++dctx->no >= rq->outpages || !rq->outputsize) {
|
|
|
|
|
erofs_err(sb, "insufficient space for decompressed data");
|
|
|
|
|
return -EFSCORRUPTED;
|
|
|
|
|
}
|
|
|
|
|
@@ -401,7 +378,7 @@ int z_erofs_stream_switch_bufs(struct z_erofs_stream_dctx *dctx, void **dst,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (dctx->inbuf_pos == dctx->inbuf_sz && rq->inputsize) {
|
|
|
|
|
if (++dctx->ni >= dctx->inpages) {
|
|
|
|
|
if (++dctx->ni >= rq->inpages) {
|
|
|
|
|
erofs_err(sb, "invalid compressed data");
|
|
|
|
|
return -EFSCORRUPTED;
|
|
|
|
|
}
|
|
|
|
|
@@ -434,7 +411,7 @@ int z_erofs_stream_switch_bufs(struct z_erofs_stream_dctx *dctx, void **dst,
|
|
|
|
|
dctx->bounced = true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (j = dctx->ni + 1; j < dctx->inpages; ++j) {
|
|
|
|
|
for (j = dctx->ni + 1; j < rq->inpages; ++j) {
|
|
|
|
|
if (rq->out[dctx->no] != rq->in[j])
|
|
|
|
|
continue;
|
|
|
|
|
tmppage = erofs_allocpage(pgpl, rq->gfp);
|
|
|
|
|
|