@@ -18,6 +18,8 @@
#include <linux/pagemap.h>
#include <linux/bio.h>
#include <linux/refcount.h>
+#include <crypto/acompress.h>
+#include <linux/scatterlist.h>
#include "compression.h"
/* workspace buffer size for s390 zlib hardware support */
@@ -33,6 +35,201 @@ struct workspace {
static struct workspace_manager wsm;
+static int acomp_comp_pages(struct address_space *mapping, u64 start,
+ unsigned long len, struct page **pages,
+ unsigned long *out_pages,
+ unsigned long *total_in,
+ unsigned long *total_out)
+{
+ unsigned int nr_src_pages = 0, nr_dst_pages = 0, nr_pages = 0;
+ struct sg_table in_sg = { 0 }, out_sg = { 0 };
+ struct page *in_page, *out_page, **in_pages;
+ struct crypto_acomp *tfm = NULL;
+ struct acomp_req *req = NULL;
+ struct crypto_wait wait;
+ int ret, i;
+
+ nr_src_pages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+ in_pages = kcalloc(nr_src_pages, sizeof(struct page *), GFP_KERNEL);
+ if (!in_pages) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ for (i = 0; i < nr_src_pages; i++) {
+ in_page = find_get_page(mapping, start >> PAGE_SHIFT);
+ out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
+ if (!in_page || !out_page) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ in_pages[i] = in_page;
+ pages[i] = out_page;
+ nr_dst_pages += 1;
+ start += PAGE_SIZE;
+ }
+
+ ret = sg_alloc_table_from_pages(&in_sg, in_pages, nr_src_pages, 0,
+ nr_src_pages << PAGE_SHIFT, GFP_KERNEL);
+ if (ret)
+ goto out;
+
+ ret = sg_alloc_table_from_pages(&out_sg, pages, nr_dst_pages, 0,
+ nr_dst_pages << PAGE_SHIFT, GFP_KERNEL);
+ if (ret)
+ goto out;
+
+ crypto_init_wait(&wait);
+ tfm = crypto_alloc_acomp("zlib-deflate", 0, 0);
+ if (IS_ERR(tfm)) {
+ ret = PTR_ERR(tfm);
+ goto out;
+ }
+
+ req = acomp_request_alloc(tfm);
+ if (!req) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ acomp_request_set_params(req, in_sg.sgl, out_sg.sgl, len,
+ nr_dst_pages << PAGE_SHIFT);
+ acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ crypto_req_done, &wait);
+
+ ret = crypto_wait_req(crypto_acomp_compress(req), &wait);
+ if (ret)
+ goto out;
+
+ *total_in = len;
+ *total_out = req->dlen;
+ nr_pages = (*total_out + PAGE_SIZE - 1) >> PAGE_SHIFT;
+
+out:
+ sg_free_table(&in_sg);
+ sg_free_table(&out_sg);
+
+ if (in_pages) {
+ for (i = 0; i < nr_src_pages; i++)
+ put_page(in_pages[i]);
+ kfree(in_pages);
+ }
+
+ /* free un-used out pages */
+ for (i = nr_pages; i < nr_dst_pages; i++)
+ put_page(pages[i]);
+
+ if (req)
+ acomp_request_free(req);
+
+ if (tfm)
+ crypto_free_acomp(tfm);
+
+ *out_pages = nr_pages;
+
+ return ret;
+}
+
+static int acomp_zlib_decomp_bio(struct page **in_pages,
+ struct compressed_bio *cb, size_t srclen,
+ unsigned long total_pages_in)
+{
+ unsigned int nr_dst_pages = BTRFS_MAX_COMPRESSED_PAGES;
+ struct sg_table in_sg = { 0 }, out_sg = { 0 };
+ struct bio *orig_bio = &cb->orig_bbio->bio;
+ char *data_out = NULL, *bv_buf = NULL;
+ int copy_len = 0, bytes_left = 0;
+ struct crypto_acomp *tfm = NULL;
+ struct page **out_pages = NULL;
+ struct acomp_req *req = NULL;
+ struct crypto_wait wait;
+ struct bio_vec bvec;
+ int ret, i = 0;
+
+ ret = sg_alloc_table_from_pages(&in_sg, in_pages, total_pages_in,
+ 0, srclen, GFP_KERNEL);
+ if (ret)
+ goto out;
+
+ out_pages = kcalloc(nr_dst_pages, sizeof(struct page *), GFP_KERNEL);
+ if (!out_pages) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ for (i = 0; i < nr_dst_pages; i++) {
+ out_pages[i] = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
+ if (!out_pages[i]) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ }
+
+ ret = sg_alloc_table_from_pages(&out_sg, out_pages, nr_dst_pages, 0,
+ nr_dst_pages << PAGE_SHIFT, GFP_KERNEL);
+ if (ret)
+ goto out;
+
+ crypto_init_wait(&wait);
+ tfm = crypto_alloc_acomp("zlib-deflate", 0, 0);
+ if (IS_ERR(tfm)) {
+ ret = PTR_ERR(tfm);
+ goto out;
+ }
+
+ req = acomp_request_alloc(tfm);
+ if (!req) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ acomp_request_set_params(req, in_sg.sgl, out_sg.sgl, srclen,
+ nr_dst_pages << PAGE_SHIFT);
+ acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ crypto_req_done, &wait);
+
+ ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
+ if (ret)
+ goto out;
+
+ /* Copy decompressed buffer to bio pages */
+ bytes_left = req->dlen;
+ for (i = 0; i < nr_dst_pages; i++) {
+ copy_len = bytes_left > PAGE_SIZE ? PAGE_SIZE : bytes_left;
+ data_out = kmap_local_page(out_pages[i]);
+
+ bvec = bio_iter_iovec(orig_bio, orig_bio->bi_iter);
+ bv_buf = kmap_local_page(bvec.bv_page);
+ memcpy(bv_buf, data_out, copy_len);
+ kunmap_local(bv_buf);
+
+ bio_advance(orig_bio, copy_len);
+ if (!orig_bio->bi_iter.bi_size)
+ break;
+ bytes_left -= copy_len;
+ if (bytes_left <= 0)
+ break;
+ }
+out:
+ sg_free_table(&in_sg);
+ sg_free_table(&out_sg);
+
+ if (out_pages) {
+ for (i = 0; i < nr_dst_pages; i++) {
+ if (out_pages[i])
+ put_page(out_pages[i]);
+ }
+ kfree(out_pages);
+ }
+
+ if (req)
+ acomp_request_free(req);
+ if (tfm)
+ crypto_free_acomp(tfm);
+
+ return ret;
+}
+
struct list_head *zlib_get_workspace(unsigned int level)
{
struct list_head *ws = btrfs_get_workspace(BTRFS_COMPRESS_ZLIB, level);
@@ -108,6 +305,15 @@ int zlib_compress_pages(struct list_head *ws, struct address_space *mapping,
unsigned long nr_dest_pages = *out_pages;
const unsigned long max_out = nr_dest_pages * PAGE_SIZE;
+ if (crypto_has_acomp("zlib-deflate", 0, 0)) {
+ ret = acomp_comp_pages(mapping, start, len, pages, out_pages,
+ total_in, total_out);
+ if (!ret)
+ return ret;
+
+ pr_warn("BTRFS: acomp compression failed: ret = %d\n", ret);
+ /* Fallback to SW implementation if HW compression failed */
+ }
*out_pages = 0;
*total_out = 0;
*total_in = 0;
@@ -281,6 +487,16 @@ int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
unsigned long buf_start;
struct page **pages_in = cb->compressed_pages;
+ if (crypto_has_acomp("zlib-deflate", 0, 0)) {
+ ret = acomp_zlib_decomp_bio(pages_in, cb, srclen,
+ total_pages_in);
+ if (!ret)
+ return ret;
+
+ pr_warn("BTRFS: acomp decompression failed, ret=%d\n", ret);
+ /* Fallback to SW implementation if HW decompression failed */
+ }
+
data_in = kmap_local_page(pages_in[page_in_index]);
workspace->strm.next_in = data_in;
workspace->strm.avail_in = min_t(size_t, srclen, PAGE_SIZE);