Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:Ledest:erlang:18
lz4-erlang
0001-Accept-iovec-as-input-for-lz4f-compress_up...
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 0001-Accept-iovec-as-input-for-lz4f-compress_update-2.patch of Package lz4-erlang
From 42661114e15c6ebe6725c83a4b01599f1de32e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= <essen@ninenines.eu> Date: Tue, 17 Sep 2019 12:45:46 +0200 Subject: [PATCH] Accept iovec() as input for lz4f:compress_update/2 iovec() is [binary()]. --- c_src/lz4_erlang.h | 1 + c_src/lz4f.c | 43 +++++++++++++++++++++++++++++++------------ src/lz4f.erl | 14 ++++++++++++-- test/lz4f_SUITE.erl | 18 ++++++++++++++++++ 4 files changed, 62 insertions(+), 14 deletions(-) diff --git a/c_src/lz4_erlang.h b/c_src/lz4_erlang.h index f233395..2ab6db7 100644 --- a/c_src/lz4_erlang.h +++ b/c_src/lz4_erlang.h @@ -26,6 +26,7 @@ #define NIF_ATOMS(A) \ A(ok) \ A(done) \ + A(continue) \ A(enomem) \ A(true) \ A(false) \ diff --git a/c_src/lz4f.c b/c_src/lz4f.c index e23baee..1aeaa51 100644 --- a/c_src/lz4f.c +++ b/c_src/lz4f.c @@ -183,28 +183,42 @@ NIF_FUNCTION(lz4f_compress_begin) NIF_FUNCTION(lz4f_compress_update) { void* cctx_res; - size_t dstCapacity, dstSize; - ErlNifBinary srcBin, dstBin; + LZ4F_cctx* cctx; + ErlNifIOVec prealloc, *iovec = &prealloc; + size_t dstCapacity, dstSize = 0, chunkSize; + ErlNifBinary dstBin; + ERL_NIF_TERM compressed, tail; + int i; BADARG_IF(!enif_get_resource(env, argv[0], res_LZ4F_cctx, &cctx_res)); - BADARG_IF(!enif_inspect_binary(env, argv[1], &srcBin)); + BADARG_IF(!enif_inspect_iovec(env, 256, argv[1], &tail, &iovec)); // @todo We pass NULL because we don't currently keep the preferences // setup when the user began the compression. It might be done later // as an optimization. - dstCapacity = LZ4F_compressBound(srcBin.size, NULL); + dstCapacity = LZ4F_compressBound(iovec->size, NULL); if (!enif_alloc_binary(dstCapacity, &dstBin)) return enif_raise_exception(env, atom_enomem); - // We pass NULL because we can't guarantee that the source binary - // data will remain for future calls. It may be garbage collected. - dstSize = LZ4F_compressUpdate(NIF_RES_GET(LZ4F_cctx, cctx_res), - dstBin.data, dstCapacity, srcBin.data, srcBin.size, NULL); + cctx = NIF_RES_GET(LZ4F_cctx, cctx_res); - if (LZ4F_isError(dstSize)) { - enif_release_binary(&dstBin); - return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(dstSize))); + for (i = 0; i < iovec->iovcnt; i++) { + // We pass NULL because we can't guarantee that the source binary + // data will remain for future calls. It may be garbage collected. + chunkSize = LZ4F_compressUpdate(cctx, + dstBin.data + dstSize, + dstCapacity - dstSize, + iovec->iov[i].iov_base, + iovec->iov[i].iov_len, + NULL); + + if (LZ4F_isError(chunkSize)) { + enif_release_binary(&dstBin); + return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(dstSize))); + } + + dstSize += chunkSize; } if (!enif_realloc_binary(&dstBin, dstSize)) { @@ -212,7 +226,12 @@ NIF_FUNCTION(lz4f_compress_update) return enif_raise_exception(env, atom_enomem); } - return enif_make_binary(env, &dstBin); + compressed = enif_make_binary(env, &dstBin); + + if (enif_is_empty_list(env, tail)) + return enif_make_tuple2(env, atom_ok, compressed); + + return enif_make_tuple3(env, atom_continue, compressed, tail); } NIF_FUNCTION(lz4f_flush) diff --git a/src/lz4f.erl b/src/lz4f.erl index e9e047b..055be59 100644 --- a/src/lz4f.erl +++ b/src/lz4f.erl @@ -76,9 +76,19 @@ compress_begin(Cctx) -> compress_begin(Cctx, Opts) -> lz4_nif:lz4f_compress_begin(Cctx, Opts). --spec compress_update(cctx(), binary()) -> binary(). +-spec compress_update(cctx(), binary() | [binary()]) -> iolist(). +compress_update(Cctx, Data) when is_list(Data) -> + compress_update_loop(Cctx, Data, []); compress_update(Cctx, Data) -> - lz4_nif:lz4f_compress_update(Cctx, Data). + compress_update_loop(Cctx, [Data], []). + +compress_update_loop(Cctx, Data, Acc) -> + case lz4_nif:lz4f_compress_update(Cctx, Data) of + {ok, Compressed} -> + lists:reverse([Compressed|Acc]); + {continue, Compressed, Rest} -> + compress_update_loop(Cctx, Rest, [Compressed|Acc]) + end. -spec flush(cctx()) -> binary(). flush(Cctx) -> diff --git a/test/lz4f_SUITE.erl b/test/lz4f_SUITE.erl index 875572a..2187ede 100644 --- a/test/lz4f_SUITE.erl +++ b/test/lz4f_SUITE.erl @@ -46,6 +46,19 @@ compress_update(Config) -> File = iolist_to_binary(lz4f:decompress(Compressed)), ok. +compress_update_iovec(Config) -> + doc("Stream compress with lz4f:compress_update/2 and then decompress back. " + "The compression is done on iovec() data rather than binary()."), + {ok, File} = file:read_file(config(test_file, Config)), + Chunks = do_slice(File), + Ctx = lz4f:create_compression_context(), + Begin = lz4f:compress_begin(Ctx), + CompressedChunks = [lz4f:compress_update(Ctx, do_slice_iovec(C)) || C <- Chunks], + End = lz4f:compress_end(Ctx), + Compressed = iolist_to_binary([Begin, CompressedChunks, End]), + File = iolist_to_binary(lz4f:decompress(Compressed)), + ok. + compress_update_opts(Config) -> doc("Stream compress with lz4f:compress_update/2 and then decompress back. " "The compression is initialized with lz4f:compress_begin/2."), @@ -112,3 +125,8 @@ do_slice(<<Bin:1000000/binary, R/bits>>) -> [Bin|do_slice(R)]; do_slice(Bin) -> [Bin]. + +do_slice_iovec(<<Bin:1000/binary, R/bits>>) -> + [Bin|do_slice_iovec(R)]; +do_slice_iovec(Bin) -> + [Bin]. -- 2.35.3
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
