linux/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2022 Intel Corporation */
#include <linux/crypto.h>
#include <crypto/acompress.h>
#include <crypto/internal/acompress.h>
#include <crypto/scatterwalk.h>
#include <linux/dma-mapping.h>
#include <linux/workqueue.h>
#include <linux/zstd.h>
#include "adf_accel_devices.h"
#include "adf_common_drv.h"
#include "adf_dc.h"
#include "qat_bl.h"
#include "qat_comp_req.h"
#include "qat_compression.h"
#include "qat_algs_send.h"
#include "qat_comp_zstd_utils.h"

#define QAT_ZSTD_SCRATCH_SIZE		524288
#define QAT_ZSTD_MAX_BLOCK_SIZE		65535
#define QAT_ZSTD_MAX_CONTENT_SIZE	4096
#define QAT_LZ4S_MIN_INPUT_SIZE		8192
#define QAT_LZ4S_MAX_OUTPUT_SIZE	QAT_ZSTD_SCRATCH_SIZE
#define QAT_MAX_SEQUENCES		(128 * 1024)

static DEFINE_MUTEX(algs_lock);
static unsigned int active_devs_deflate;
static unsigned int active_devs_lz4s;
static unsigned int active_devs_zstd;

struct qat_zstd_scratch {
	size_t		cctx_buffer_size;
	void		*lz4s;
	void		*literals;
	void		*out_seqs;
	void		*workspace;
	ZSTD_CCtx	*ctx;
};

static void *qat_zstd_alloc_scratch(void)
{
	struct qat_zstd_scratch *scratch;
	ZSTD_parameters params;
	size_t cctx_size;
	ZSTD_CCtx *ctx;
	size_t zret;
	int ret;

	ret = -ENOMEM;
	scratch = kzalloc_obj(*scratch);
	if (!scratch)
		return ERR_PTR(ret);

	scratch->lz4s = kvmalloc(QAT_ZSTD_SCRATCH_SIZE, GFP_KERNEL);
	if (!scratch->lz4s)
		goto error;

	scratch->literals = kvmalloc(QAT_ZSTD_SCRATCH_SIZE, GFP_KERNEL);
	if (!scratch->literals)
		goto error;

	scratch->out_seqs = kvzalloc_objs(ZSTD_Sequence, QAT_MAX_SEQUENCES);
	if (!scratch->out_seqs)
		goto error;

	params = zstd_get_params(zstd_max_clevel(), QAT_ZSTD_SCRATCH_SIZE);
	cctx_size = zstd_cctx_workspace_bound(&params.cParams);

	scratch->workspace = kvmalloc(cctx_size, GFP_KERNEL | __GFP_ZERO);
	if (!scratch->workspace)
		goto error;

	ret = -EINVAL;
	ctx = zstd_init_cctx(scratch->workspace, cctx_size);
	if (!ctx)
		goto error;

	scratch->ctx = ctx;
	scratch->cctx_buffer_size = cctx_size;

	zret = zstd_cctx_set_param(ctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
	if (zstd_is_error(zret))
		goto error;

	return scratch;

error:
	kvfree(scratch->lz4s);
	kvfree(scratch->literals);
	kvfree(scratch->out_seqs);
	kvfree(scratch->workspace);
	kfree(scratch);
	return ERR_PTR(ret);
}

static void qat_zstd_free_scratch(void *ctx)
{
	struct qat_zstd_scratch *scratch = ctx;

	if (!scratch)
		return;

	kvfree(scratch->lz4s);
	kvfree(scratch->literals);
	kvfree(scratch->out_seqs);
	kvfree(scratch->workspace);
	kfree(scratch);
}

static struct crypto_acomp_streams qat_zstd_streams = {
	.alloc_ctx = qat_zstd_alloc_scratch,
	.free_ctx = qat_zstd_free_scratch,
};

enum direction {
	DECOMPRESSION = 0,
	COMPRESSION = 1,
};

struct qat_compression_req;

struct qat_callback_params {
	unsigned int produced;
	unsigned int dlen;
	bool plain;
};

struct qat_compression_ctx {
	u8 comp_ctx[QAT_COMP_CTX_SIZE];
	struct qat_compression_instance *inst;
	int (*qat_comp_callback)(struct qat_compression_req *qat_req, void *resp,
				 struct qat_callback_params *params);
	struct crypto_acomp *ftfm;
};

struct qat_compression_req {
	u8 req[QAT_COMP_REQ_SIZE];
	struct qat_compression_ctx *qat_compression_ctx;
	struct acomp_req *acompress_req;
	struct qat_request_buffs buf;
	enum direction dir;
	int actual_dlen;
	struct qat_alg_req alg_req;
};

static int qat_alg_send_dc_message(struct qat_compression_req *qat_req,
				   struct qat_compression_instance *inst,
				   struct crypto_async_request *base)
{
	struct qat_alg_req *alg_req = &qat_req->alg_req;

	alg_req->fw_req = (u32 *)&qat_req->req;
	alg_req->tx_ring = inst->dc_tx;
	alg_req->base = base;
	alg_req->backlog = &inst->backlog;

	return qat_alg_send_message(alg_req);
}

static void qat_comp_generic_callback(struct qat_compression_req *qat_req,
				      void *resp)
{
	struct acomp_req *areq = qat_req->acompress_req;
	struct qat_compression_ctx *ctx = qat_req->qat_compression_ctx;
	struct adf_accel_dev *accel_dev = ctx->inst->accel_dev;
	struct crypto_acomp *tfm = crypto_acomp_reqtfm(areq);
	struct qat_compression_instance *inst = ctx->inst;
	struct qat_callback_params params = { };
	int consumed, produced;
	s8 cmp_err, xlt_err;
	int res = -EBADMSG;
	int status;
	u8 cnv;

	status = qat_comp_get_cmp_status(resp);
	status |= qat_comp_get_xlt_status(resp);
	cmp_err = qat_comp_get_cmp_err(resp);
	xlt_err = qat_comp_get_xlt_err(resp);

	consumed = qat_comp_get_consumed_ctr(resp);
	produced = qat_comp_get_produced_ctr(resp);

	/* Cache parameters for algorithm specific callback */
	params.produced = produced;
	params.dlen = areq->dlen;

	dev_dbg(&GET_DEV(accel_dev),
		"[%s][%s][%s] slen = %8d dlen = %8d consumed = %8d produced = %8d cmp_err = %3d xlt_err = %3d",
		crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)),
		qat_req->dir == COMPRESSION ? "comp  " : "decomp",
		status ? "ERR" : "OK ",
		areq->slen, areq->dlen, consumed, produced, cmp_err, xlt_err);

	if (unlikely(status != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) {
		if (cmp_err == ERR_CODE_OVERFLOW_ERROR || xlt_err == ERR_CODE_OVERFLOW_ERROR)
			res = -E2BIG;

		areq->dlen = 0;
		goto end;
	}

	if (qat_req->dir == COMPRESSION) {
		cnv = qat_comp_get_cmp_cnv_flag(resp);
		if (unlikely(!cnv)) {
			dev_err(&GET_DEV(accel_dev),
				"Verified compression not supported\n");
			areq->dlen = 0;
			goto end;
		}

		if (unlikely(produced > qat_req->actual_dlen)) {
			memset(inst->dc_data->ovf_buff, 0,
			       inst->dc_data->ovf_buff_sz);
			dev_dbg(&GET_DEV(accel_dev),
				"Actual buffer overflow: produced=%d, dlen=%d\n",
				produced, qat_req->actual_dlen);

			res = -E2BIG;
			areq->dlen = 0;
			goto end;
		}

		params.plain = !!qat_comp_get_cmp_uncomp_flag(resp);
	}

	res = 0;
	areq->dlen = produced;

	if (ctx->qat_comp_callback)
		res = ctx->qat_comp_callback(qat_req, resp, &params);

end:
	qat_bl_free_bufl(accel_dev, &qat_req->buf);
	acomp_request_complete(areq, res);
	qat_alg_send_backlog(qat_req->alg_req.backlog);
}

void qat_comp_alg_callback(void *resp)
{
	struct qat_compression_req *qat_req =
			(void *)(__force long)qat_comp_get_opaque(resp);

	qat_comp_generic_callback(qat_req, resp);
}

static int qat_comp_alg_init_tfm(struct crypto_acomp *acomp_tfm, int alg)
{
	struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
	struct qat_compression_instance *inst;
	int node, ret;

	if (tfm->node == NUMA_NO_NODE)
		node = numa_node_id();
	else
		node = tfm->node;

	memset(ctx, 0, sizeof(*ctx));
	inst = qat_compression_get_instance_node(node, alg);
	if (!inst)
		return -EINVAL;
	ctx->inst = inst;

	ret = qat_comp_build_ctx(inst->accel_dev, ctx->comp_ctx, alg);
	if (ret) {
		qat_compression_put_instance(inst);
		memset(ctx, 0, sizeof(*ctx));
	}

	return ret;
}

static int qat_comp_alg_deflate_init_tfm(struct crypto_acomp *acomp_tfm)
{
	return qat_comp_alg_init_tfm(acomp_tfm, QAT_DEFLATE);
}

static void qat_comp_alg_exit_tfm(struct crypto_acomp *acomp_tfm)
{
	struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);

	qat_compression_put_instance(ctx->inst);
	memset(ctx, 0, sizeof(*ctx));
}

static int qat_comp_alg_compress_decompress(struct acomp_req *areq, enum direction dir,
					    unsigned int shdr, unsigned int sftr,
					    unsigned int dhdr, unsigned int dftr)
{
	struct qat_compression_req *qat_req = acomp_request_ctx(areq);
	struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(areq);
	struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
	struct qat_compression_instance *inst = ctx->inst;
	gfp_t f = qat_algs_alloc_flags(&areq->base);
	struct qat_sgl_to_bufl_params params = {0};
	int slen = areq->slen - shdr - sftr;
	int dlen = areq->dlen - dhdr - dftr;
	dma_addr_t sfbuf, dfbuf;
	u8 *req = qat_req->req;
	size_t ovf_buff_sz;
	int ret;

	params.sskip = shdr;
	params.dskip = dhdr;

	if (!areq->src || !slen)
		return -EINVAL;

	if (!areq->dst || !dlen)
		return -EINVAL;

	if (dir == COMPRESSION) {
		params.extra_dst_buff = inst->dc_data->ovf_buff_p;
		ovf_buff_sz = inst->dc_data->ovf_buff_sz;
		params.sz_extra_dst_buff = ovf_buff_sz;
	}

	ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst,
				 &qat_req->buf, &params, f);
	if (unlikely(ret))
		return ret;

	sfbuf = qat_req->buf.blp;
	dfbuf = qat_req->buf.bloutp;
	qat_req->qat_compression_ctx = ctx;
	qat_req->acompress_req = areq;
	qat_req->dir = dir;

	if (dir == COMPRESSION) {
		qat_req->actual_dlen = dlen;
		dlen += ovf_buff_sz;
		qat_comp_create_compression_req(ctx->comp_ctx, req,
						(u64)(__force long)sfbuf, slen,
						(u64)(__force long)dfbuf, dlen,
						(u64)(__force long)qat_req);
	} else {
		qat_comp_create_decompression_req(ctx->comp_ctx, req,
						  (u64)(__force long)sfbuf, slen,
						  (u64)(__force long)dfbuf, dlen,
						  (u64)(__force long)qat_req);
	}

	ret = qat_alg_send_dc_message(qat_req, inst, &areq->base);
	if (ret == -ENOSPC)
		qat_bl_free_bufl(inst->accel_dev, &qat_req->buf);

	return ret;
}

static int qat_comp_alg_compress(struct acomp_req *req)
{
	return qat_comp_alg_compress_decompress(req, COMPRESSION, 0, 0, 0, 0);
}

static int qat_comp_alg_decompress(struct acomp_req *req)
{
	return qat_comp_alg_compress_decompress(req, DECOMPRESSION, 0, 0, 0, 0);
}

static int qat_comp_alg_zstd_decompress(struct acomp_req *req)
{
	struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(req);
	struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
	struct acomp_req *nreq = acomp_request_ctx(req);
	zstd_frame_header header;
	void *buffer;
	size_t zret;
	int ret;

	buffer = kmap_local_page(sg_page(req->src)) + req->src->offset;
	zret = zstd_get_frame_header(&header, buffer, req->src->length);
	kunmap_local(buffer);
	if (zret) {
		dev_err(&GET_DEV(ctx->inst->accel_dev),
			"ZSTD-compressed data has an incomplete frame header\n");
		return -EINVAL;
	}

	if (header.windowSize > QAT_ZSTD_MAX_BLOCK_SIZE ||
	    header.frameContentSize >= QAT_ZSTD_MAX_CONTENT_SIZE) {
		dev_dbg(&GET_DEV(ctx->inst->accel_dev), "Window size=0x%llx\n",
			header.windowSize);

		memcpy(nreq, req, sizeof(*req));
		acomp_request_set_tfm(nreq, ctx->ftfm);

		ret = crypto_acomp_decompress(nreq);
		req->dlen = nreq->dlen;

		return ret;
	}

	return qat_comp_alg_compress_decompress(req, DECOMPRESSION, 0, 0, 0, 0);
}

static int qat_comp_lz4s_zstd_callback(struct qat_compression_req *qat_req, void *resp,
				       struct qat_callback_params *params)
{
	struct qat_compression_ctx *qat_ctx = qat_req->qat_compression_ctx;
	struct acomp_req *areq = qat_req->acompress_req;
	struct qat_zstd_scratch *scratch;
	struct crypto_acomp_stream *s;
	unsigned int lit_len = 0;
	ZSTD_Sequence *out_seqs;
	void *lz4s, *zstd;
	size_t comp_size;
	ZSTD_CCtx *ctx;
	void *literals;
	int seq_count;
	int ret = 0;

	if (params->produced + QAT_ZSTD_LIT_COPY_LEN > QAT_ZSTD_SCRATCH_SIZE) {
		dev_dbg(&GET_DEV(qat_ctx->inst->accel_dev),
			"LZ4s-ZSTD: produced size (%u) + COPY_SIZE > QAT_ZSTD_SCRATCH_SIZE (%u)\n",
			params->produced, QAT_ZSTD_SCRATCH_SIZE);
		areq->dlen = 0;
		return -E2BIG;
	}

	s = crypto_acomp_lock_stream_bh(&qat_zstd_streams);
	scratch = s->ctx;

	lz4s = scratch->lz4s;
	zstd = lz4s;  /* Output buffer is same as lz4s */
	out_seqs = scratch->out_seqs;
	ctx = scratch->ctx;
	literals = scratch->literals;

	if (likely(!params->plain)) {
		if (likely(sg_nents(areq->dst) == 1)) {
			zstd = sg_virt(areq->dst);
			lz4s = zstd;
		} else {
			memcpy_from_sglist(lz4s, areq->dst, 0, params->produced);
		}

		seq_count = qat_alg_dec_lz4s(out_seqs, QAT_MAX_SEQUENCES, lz4s,
					     params->produced, literals, &lit_len);
		if (seq_count < 0) {
			ret = seq_count;
			comp_size = 0;
			goto out;
		}
	} else {
		out_seqs[0].litLength = areq->slen;
		out_seqs[0].offset = 0;
		out_seqs[0].matchLength = 0;

		seq_count = 1;
	}

	comp_size = zstd_compress_sequences_and_literals(ctx, zstd, params->dlen,
							 out_seqs, seq_count,
							 literals, lit_len,
							 QAT_ZSTD_SCRATCH_SIZE,
							 areq->slen);
	if (zstd_is_error(comp_size)) {
		if (comp_size == ZSTD_error_cannotProduce_uncompressedBlock)
			ret = -E2BIG;
		else
			ret = -EOPNOTSUPP;

		comp_size = 0;
		goto out;
	}

	if (comp_size > params->dlen) {
		dev_dbg(&GET_DEV(qat_ctx->inst->accel_dev),
			"LZ4s-ZSTD: compressed_size (%u) > output buffer size (%u)\n",
			(unsigned int)comp_size, params->dlen);
		ret = -EOVERFLOW;
		goto out;
	}

	if (unlikely(sg_nents(areq->dst) != 1))
		memcpy_to_sglist(areq->dst, 0, zstd, comp_size);

out:
	areq->dlen = comp_size;
	crypto_acomp_unlock_stream_bh(s);

	return ret;
}

static int qat_comp_alg_lz4s_zstd_init_tfm(struct crypto_acomp *acomp_tfm)
{
	struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
	int reqsize;
	int ret;

	/* qat_comp_alg_init_tfm() wipes out the ctx */
	ret = qat_comp_alg_init_tfm(acomp_tfm, QAT_LZ4S);
	if (ret)
		return ret;

	ctx->ftfm = crypto_alloc_acomp_node("zstd", 0, CRYPTO_ALG_NEED_FALLBACK,
					    tfm->node);
	if (IS_ERR(ctx->ftfm)) {
		qat_comp_alg_exit_tfm(acomp_tfm);
		return PTR_ERR(ctx->ftfm);
	}

	reqsize = max(sizeof(struct qat_compression_req),
		      sizeof(struct acomp_req) + crypto_acomp_reqsize(ctx->ftfm));

	acomp_tfm->reqsize = reqsize;

	ctx->qat_comp_callback = qat_comp_lz4s_zstd_callback;

	return 0;
}

static int qat_comp_alg_zstd_init_tfm(struct crypto_acomp *acomp_tfm)
{
	struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
	int reqsize;
	int ret;

	/* qat_comp_alg_init_tfm() wipes out the ctx */
	ret = qat_comp_alg_init_tfm(acomp_tfm, QAT_ZSTD);
	if (ret)
		return ret;

	ctx->ftfm = crypto_alloc_acomp_node("zstd", 0, CRYPTO_ALG_NEED_FALLBACK,
					    tfm->node);
	if (IS_ERR(ctx->ftfm)) {
		qat_comp_alg_exit_tfm(acomp_tfm);
		return PTR_ERR(ctx->ftfm);
	}

	reqsize = max(sizeof(struct qat_compression_req),
		      sizeof(struct acomp_req) + crypto_acomp_reqsize(ctx->ftfm));

	acomp_tfm->reqsize = reqsize;

	return 0;
}

static void qat_comp_alg_zstd_exit_tfm(struct crypto_acomp *acomp_tfm)
{
	struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);

	if (ctx->ftfm)
		crypto_free_acomp(ctx->ftfm);

	qat_comp_alg_exit_tfm(acomp_tfm);
}

static int qat_comp_alg_lz4s_zstd_compress(struct acomp_req *req)
{
	struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(req);
	struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
	struct acomp_req *nreq = acomp_request_ctx(req);
	int ret;

	if (req->slen >= QAT_LZ4S_MIN_INPUT_SIZE && req->dlen >= QAT_LZ4S_MIN_INPUT_SIZE &&
	    req->slen <= QAT_LZ4S_MAX_OUTPUT_SIZE && req->dlen <= QAT_LZ4S_MAX_OUTPUT_SIZE)
		return qat_comp_alg_compress(req);

	memcpy(nreq, req, sizeof(*req));
	acomp_request_set_tfm(nreq, ctx->ftfm);

	ret = crypto_acomp_compress(nreq);
	req->dlen = nreq->dlen;

	return ret;
}

static int qat_comp_alg_sw_decompress(struct acomp_req *req)
{
	struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(req);
	struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
	struct acomp_req *nreq = acomp_request_ctx(req);
	int ret;

	memcpy(nreq, req, sizeof(*req));
	acomp_request_set_tfm(nreq, ctx->ftfm);

	ret = crypto_acomp_decompress(nreq);
	req->dlen = nreq->dlen;

	return ret;
}

static struct acomp_alg qat_acomp_deflate[] = { {
	.base = {
		.cra_name = "deflate",
		.cra_driver_name = "qat_deflate",
		.cra_priority = 4001,
		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
		.cra_ctxsize = sizeof(struct qat_compression_ctx),
		.cra_reqsize = sizeof(struct qat_compression_req),
		.cra_module = THIS_MODULE,
	},
	.init = qat_comp_alg_deflate_init_tfm,
	.exit = qat_comp_alg_exit_tfm,
	.compress = qat_comp_alg_compress,
	.decompress = qat_comp_alg_decompress,
}};

static struct acomp_alg qat_acomp_zstd_lz4s = {
	.base = {
		.cra_name = "zstd",
		.cra_driver_name = "qat_zstd",
		.cra_priority = 4001,
		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
			     CRYPTO_ALG_NEED_FALLBACK,
		.cra_reqsize = sizeof(struct qat_compression_req),
		.cra_ctxsize = sizeof(struct qat_compression_ctx),
		.cra_module = THIS_MODULE,
	},
	.init = qat_comp_alg_lz4s_zstd_init_tfm,
	.exit = qat_comp_alg_zstd_exit_tfm,
	.compress = qat_comp_alg_lz4s_zstd_compress,
	.decompress = qat_comp_alg_sw_decompress,
};

static struct acomp_alg qat_acomp_zstd_native = {
	.base = {
		.cra_name = "zstd",
		.cra_driver_name = "qat_zstd",
		.cra_priority = 4001,
		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
			     CRYPTO_ALG_NEED_FALLBACK,
		.cra_reqsize = sizeof(struct qat_compression_req),
		.cra_ctxsize = sizeof(struct qat_compression_ctx),
		.cra_module = THIS_MODULE,
	},
	.init = qat_comp_alg_zstd_init_tfm,
	.exit = qat_comp_alg_zstd_exit_tfm,
	.compress = qat_comp_alg_compress,
	.decompress = qat_comp_alg_zstd_decompress,
};

static int qat_comp_algs_register_deflate(void)
{
	int ret = 0;

	mutex_lock(&algs_lock);
	if (++active_devs_deflate == 1) {
		ret = crypto_register_acomps(qat_acomp_deflate,
					     ARRAY_SIZE(qat_acomp_deflate));
		if (ret)
			active_devs_deflate--;
	}
	mutex_unlock(&algs_lock);

	return ret;
}

static void qat_comp_algs_unregister_deflate(void)
{
	mutex_lock(&algs_lock);
	if (--active_devs_deflate == 0)
		crypto_unregister_acomps(qat_acomp_deflate, ARRAY_SIZE(qat_acomp_deflate));
	mutex_unlock(&algs_lock);
}

static int qat_comp_algs_register_lz4s(void)
{
	int ret = 0;

	mutex_lock(&algs_lock);
	if (++active_devs_lz4s == 1) {
		ret = crypto_acomp_alloc_streams(&qat_zstd_streams);
		if (ret) {
			active_devs_lz4s--;
			goto unlock;
		}

		ret = crypto_register_acomp(&qat_acomp_zstd_lz4s);
		if (ret) {
			crypto_acomp_free_streams(&qat_zstd_streams);
			active_devs_lz4s--;
		}
	}
unlock:
	mutex_unlock(&algs_lock);

	return ret;
}

static void qat_comp_algs_unregister_lz4s(void)
{
	mutex_lock(&algs_lock);
	if (--active_devs_lz4s == 0) {
		crypto_unregister_acomp(&qat_acomp_zstd_lz4s);
		crypto_acomp_free_streams(&qat_zstd_streams);
	}
	mutex_unlock(&algs_lock);
}

static int qat_comp_algs_register_zstd(void)
{
	int ret = 0;

	mutex_lock(&algs_lock);
	if (++active_devs_zstd == 1) {
		ret = crypto_register_acomp(&qat_acomp_zstd_native);
		if (ret)
			active_devs_zstd--;
	}
	mutex_unlock(&algs_lock);

	return ret;
}

static void qat_comp_algs_unregister_zstd(void)
{
	mutex_lock(&algs_lock);
	if (--active_devs_zstd == 0)
		crypto_unregister_acomp(&qat_acomp_zstd_native);
	mutex_unlock(&algs_lock);
}

int qat_comp_algs_register(u32 caps)
{
	int ret;

	ret = qat_comp_algs_register_deflate();
	if (ret)
		return ret;

	if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S) {
		ret = qat_comp_algs_register_lz4s();
		if (ret)
			goto err_unregister_deflate;
	}

	if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD) {
		ret = qat_comp_algs_register_zstd();
		if (ret)
			goto err_unregister_lz4s;
	}

	return ret;

err_unregister_lz4s:
	if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S)
		qat_comp_algs_unregister_lz4s();
err_unregister_deflate:
	qat_comp_algs_unregister_deflate();

	return ret;
}

void qat_comp_algs_unregister(u32 caps)
{
	qat_comp_algs_unregister_deflate();

	if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S)
		qat_comp_algs_unregister_lz4s();

	if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD)
		qat_comp_algs_unregister_zstd();
}