mirror of https://github.com/postgres/postgres
Both client-side compression and server-side compression are now supported for zstd. In addition, a backup compressed by the server using zstd can now be decompressed by the client in order to accommodate the use of -Fp. Jeevan Ladhe, with some edits by me. Discussion: http://postgr.es/m/CA+Tgmobyzfbz=gyze2_LL1ZumZunmaEKbHQxjrFkOR7APZGu-g@mail.gmail.compull/81/head
Robert Haas
4 years ago
parent
c28839c832
commit
7cf085f077
@ -0,0 +1,299 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* basebackup_zstd.c |
||||
* Basebackup sink implementing zstd compression. |
||||
* |
||||
* Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group |
||||
* |
||||
* IDENTIFICATION |
||||
* src/backend/replication/basebackup_zstd.c |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
#include "postgres.h" |
||||
|
||||
#ifdef HAVE_LIBZSTD |
||||
#include <zstd.h> |
||||
#endif |
||||
|
||||
#include "replication/basebackup_sink.h" |
||||
|
||||
#ifdef HAVE_LIBZSTD |
||||
|
||||
typedef struct bbsink_zstd |
||||
{ |
||||
/* Common information for all types of sink. */ |
||||
bbsink base; |
||||
|
||||
/* Compression level */ |
||||
int compresslevel; |
||||
|
||||
ZSTD_CCtx *cctx; |
||||
ZSTD_outBuffer zstd_outBuf; |
||||
} bbsink_zstd; |
||||
|
||||
static void bbsink_zstd_begin_backup(bbsink *sink); |
||||
static void bbsink_zstd_begin_archive(bbsink *sink, const char *archive_name); |
||||
static void bbsink_zstd_archive_contents(bbsink *sink, size_t avail_in); |
||||
static void bbsink_zstd_manifest_contents(bbsink *sink, size_t len); |
||||
static void bbsink_zstd_end_archive(bbsink *sink); |
||||
static void bbsink_zstd_cleanup(bbsink *sink); |
||||
static void bbsink_zstd_end_backup(bbsink *sink, XLogRecPtr endptr, |
||||
TimeLineID endtli); |
||||
|
||||
const bbsink_ops bbsink_zstd_ops = { |
||||
.begin_backup = bbsink_zstd_begin_backup, |
||||
.begin_archive = bbsink_zstd_begin_archive, |
||||
.archive_contents = bbsink_zstd_archive_contents, |
||||
.end_archive = bbsink_zstd_end_archive, |
||||
.begin_manifest = bbsink_forward_begin_manifest, |
||||
.manifest_contents = bbsink_zstd_manifest_contents, |
||||
.end_manifest = bbsink_forward_end_manifest, |
||||
.end_backup = bbsink_zstd_end_backup, |
||||
.cleanup = bbsink_zstd_cleanup |
||||
}; |
||||
#endif |
||||
|
||||
/*
|
||||
* Create a new basebackup sink that performs zstd compression using the |
||||
* designated compression level. |
||||
*/ |
||||
bbsink * |
||||
bbsink_zstd_new(bbsink *next, int compresslevel) |
||||
{ |
||||
#ifndef HAVE_LIBZSTD |
||||
ereport(ERROR, |
||||
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), |
||||
errmsg("zstd compression is not supported by this build"))); |
||||
return NULL; /* keep compiler quiet */ |
||||
#else |
||||
bbsink_zstd *sink; |
||||
|
||||
Assert(next != NULL); |
||||
|
||||
if (compresslevel < 0 || compresslevel > 22) |
||||
ereport(ERROR, |
||||
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), |
||||
errmsg("zstd compression level %d is out of range", |
||||
compresslevel))); |
||||
|
||||
sink = palloc0(sizeof(bbsink_zstd)); |
||||
*((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_zstd_ops; |
||||
sink->base.bbs_next = next; |
||||
sink->compresslevel = compresslevel; |
||||
|
||||
return &sink->base; |
||||
#endif |
||||
} |
||||
|
||||
#ifdef HAVE_LIBZSTD |
||||
|
||||
/*
|
||||
* Begin backup. |
||||
*/ |
||||
static void |
||||
bbsink_zstd_begin_backup(bbsink *sink) |
||||
{ |
||||
bbsink_zstd *mysink = (bbsink_zstd *) sink; |
||||
size_t output_buffer_bound; |
||||
|
||||
mysink->cctx = ZSTD_createCCtx(); |
||||
if (!mysink->cctx) |
||||
elog(ERROR, "could not create zstd compression context"); |
||||
|
||||
ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_compressionLevel, |
||||
mysink->compresslevel); |
||||
|
||||
/*
|
||||
* We need our own buffer, because we're going to pass different data to |
||||
* the next sink than what gets passed to us. |
||||
*/ |
||||
mysink->base.bbs_buffer = palloc(mysink->base.bbs_buffer_length); |
||||
|
||||
/*
|
||||
* Make sure that the next sink's bbs_buffer is big enough to accommodate |
||||
* the compressed input buffer. |
||||
*/ |
||||
output_buffer_bound = ZSTD_compressBound(mysink->base.bbs_buffer_length); |
||||
|
||||
/*
|
||||
* The buffer length is expected to be a multiple of BLCKSZ, so round up. |
||||
*/ |
||||
output_buffer_bound = output_buffer_bound + BLCKSZ - |
||||
(output_buffer_bound % BLCKSZ); |
||||
|
||||
bbsink_begin_backup(sink->bbs_next, sink->bbs_state, output_buffer_bound); |
||||
} |
||||
|
||||
/*
|
||||
* Prepare to compress the next archive. |
||||
*/ |
||||
static void |
||||
bbsink_zstd_begin_archive(bbsink *sink, const char *archive_name) |
||||
{ |
||||
bbsink_zstd *mysink = (bbsink_zstd *) sink; |
||||
char *zstd_archive_name; |
||||
|
||||
/*
|
||||
* At the start of each archive we reset the state to start a new |
||||
* compression operation. The parameters are sticky and they will stick |
||||
* around as we are resetting with option ZSTD_reset_session_only. |
||||
*/ |
||||
ZSTD_CCtx_reset(mysink->cctx, ZSTD_reset_session_only); |
||||
|
||||
mysink->zstd_outBuf.dst = mysink->base.bbs_next->bbs_buffer; |
||||
mysink->zstd_outBuf.size = mysink->base.bbs_next->bbs_buffer_length; |
||||
mysink->zstd_outBuf.pos = 0; |
||||
|
||||
/* Add ".zst" to the archive name. */ |
||||
zstd_archive_name = psprintf("%s.zst", archive_name); |
||||
Assert(sink->bbs_next != NULL); |
||||
bbsink_begin_archive(sink->bbs_next, zstd_archive_name); |
||||
pfree(zstd_archive_name); |
||||
} |
||||
|
||||
/*
|
||||
* Compress the input data to the output buffer until we run out of input |
||||
* data. Each time the output buffer falls below the compression bound for |
||||
* the input buffer, invoke the archive_contents() method for the next sink. |
||||
* |
||||
* Note that since we're compressing the input, it may very commonly happen |
||||
* that we consume all the input data without filling the output buffer. In |
||||
* that case, the compressed representation of the current input data won't |
||||
* actually be sent to the next bbsink until a later call to this function, |
||||
* or perhaps even not until bbsink_zstd_end_archive() is invoked. |
||||
*/ |
||||
static void |
||||
bbsink_zstd_archive_contents(bbsink *sink, size_t len) |
||||
{ |
||||
bbsink_zstd *mysink = (bbsink_zstd *) sink; |
||||
ZSTD_inBuffer inBuf = {mysink->base.bbs_buffer, len, 0}; |
||||
|
||||
while (inBuf.pos < inBuf.size) |
||||
{ |
||||
size_t yet_to_flush; |
||||
size_t max_needed = ZSTD_compressBound(inBuf.size - inBuf.pos); |
||||
|
||||
/*
|
||||
* If the out buffer is not left with enough space, send the output |
||||
* buffer to the next sink, and reset it. |
||||
*/ |
||||
if (mysink->zstd_outBuf.size - mysink->zstd_outBuf.pos < max_needed) |
||||
{ |
||||
bbsink_archive_contents(mysink->base.bbs_next, |
||||
mysink->zstd_outBuf.pos); |
||||
mysink->zstd_outBuf.dst = mysink->base.bbs_next->bbs_buffer; |
||||
mysink->zstd_outBuf.size = |
||||
mysink->base.bbs_next->bbs_buffer_length; |
||||
mysink->zstd_outBuf.pos = 0; |
||||
} |
||||
|
||||
yet_to_flush = ZSTD_compressStream2(mysink->cctx, &mysink->zstd_outBuf, |
||||
&inBuf, ZSTD_e_continue); |
||||
|
||||
if (ZSTD_isError(yet_to_flush)) |
||||
elog(ERROR, |
||||
"could not compress data: %s", |
||||
ZSTD_getErrorName(yet_to_flush)); |
||||
} |
||||
} |
||||
|
||||
/*
|
||||
* There might be some data inside zstd's internal buffers; we need to get that |
||||
* flushed out, also end the zstd frame and then get that forwarded to the |
||||
* successor sink as archive content. |
||||
* |
||||
* Then we can end processing for this archive. |
||||
*/ |
||||
static void |
||||
bbsink_zstd_end_archive(bbsink *sink) |
||||
{ |
||||
bbsink_zstd *mysink = (bbsink_zstd *) sink; |
||||
size_t yet_to_flush; |
||||
|
||||
do |
||||
{ |
||||
ZSTD_inBuffer in = {NULL, 0, 0}; |
||||
size_t max_needed = ZSTD_compressBound(0); |
||||
|
||||
/*
|
||||
* If the out buffer is not left with enough space, send the output |
||||
* buffer to the next sink, and reset it. |
||||
*/ |
||||
if (mysink->zstd_outBuf.size - mysink->zstd_outBuf.pos < max_needed) |
||||
{ |
||||
bbsink_archive_contents(mysink->base.bbs_next, |
||||
mysink->zstd_outBuf.pos); |
||||
mysink->zstd_outBuf.dst = mysink->base.bbs_next->bbs_buffer; |
||||
mysink->zstd_outBuf.size = |
||||
mysink->base.bbs_next->bbs_buffer_length; |
||||
mysink->zstd_outBuf.pos = 0; |
||||
} |
||||
|
||||
yet_to_flush = ZSTD_compressStream2(mysink->cctx, |
||||
&mysink->zstd_outBuf, |
||||
&in, ZSTD_e_end); |
||||
|
||||
if (ZSTD_isError(yet_to_flush)) |
||||
elog(ERROR, "could not compress data: %s", |
||||
ZSTD_getErrorName(yet_to_flush)); |
||||
|
||||
} while (yet_to_flush > 0); |
||||
|
||||
/* Make sure to pass any remaining bytes to the next sink. */ |
||||
if (mysink->zstd_outBuf.pos > 0) |
||||
bbsink_archive_contents(mysink->base.bbs_next, |
||||
mysink->zstd_outBuf.pos); |
||||
|
||||
/* Pass on the information that this archive has ended. */ |
||||
bbsink_forward_end_archive(sink); |
||||
} |
||||
|
||||
/*
|
||||
* Free the resources and context. |
||||
*/ |
||||
static void |
||||
bbsink_zstd_end_backup(bbsink *sink, XLogRecPtr endptr, |
||||
TimeLineID endtli) |
||||
{ |
||||
bbsink_zstd *mysink = (bbsink_zstd *) sink; |
||||
|
||||
/* Release the context. */ |
||||
if (mysink->cctx) |
||||
{ |
||||
ZSTD_freeCCtx(mysink->cctx); |
||||
mysink->cctx = NULL; |
||||
} |
||||
|
||||
bbsink_forward_end_backup(sink, endptr, endtli); |
||||
} |
||||
|
||||
/*
|
||||
* Manifest contents are not compressed, but we do need to copy them into |
||||
* the successor sink's buffer, because we have our own. |
||||
*/ |
||||
static void |
||||
bbsink_zstd_manifest_contents(bbsink *sink, size_t len) |
||||
{ |
||||
memcpy(sink->bbs_next->bbs_buffer, sink->bbs_buffer, len); |
||||
bbsink_manifest_contents(sink->bbs_next, len); |
||||
} |
||||
|
||||
/*
|
||||
* In case the backup fails, make sure we free any compression context that |
||||
* got allocated, so that we don't leak memory. |
||||
*/ |
||||
static void |
||||
bbsink_zstd_cleanup(bbsink *sink) |
||||
{ |
||||
bbsink_zstd *mysink = (bbsink_zstd *) sink; |
||||
|
||||
/* Release the context if not already released. */ |
||||
if (mysink->cctx) |
||||
{ |
||||
ZSTD_freeCCtx(mysink->cctx); |
||||
mysink->cctx = NULL; |
||||
} |
||||
} |
||||
|
||||
#endif |
||||
@ -0,0 +1,338 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* bbstreamer_zstd.c |
||||
* |
||||
* Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group |
||||
* |
||||
* IDENTIFICATION |
||||
* src/bin/pg_basebackup/bbstreamer_zstd.c |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
|
||||
#include "postgres_fe.h" |
||||
|
||||
#include <unistd.h> |
||||
|
||||
#ifdef HAVE_LIBZSTD |
||||
#include <zstd.h> |
||||
#endif |
||||
|
||||
#include "bbstreamer.h" |
||||
#include "common/logging.h" |
||||
|
||||
#ifdef HAVE_LIBZSTD |
||||
|
||||
typedef struct bbstreamer_zstd_frame |
||||
{ |
||||
bbstreamer base; |
||||
|
||||
ZSTD_CCtx *cctx; |
||||
ZSTD_DCtx *dctx; |
||||
ZSTD_outBuffer zstd_outBuf; |
||||
} bbstreamer_zstd_frame; |
||||
|
||||
static void bbstreamer_zstd_compressor_content(bbstreamer *streamer, |
||||
bbstreamer_member *member, |
||||
const char *data, int len, |
||||
bbstreamer_archive_context context); |
||||
static void bbstreamer_zstd_compressor_finalize(bbstreamer *streamer); |
||||
static void bbstreamer_zstd_compressor_free(bbstreamer *streamer); |
||||
|
||||
const bbstreamer_ops bbstreamer_zstd_compressor_ops = { |
||||
.content = bbstreamer_zstd_compressor_content, |
||||
.finalize = bbstreamer_zstd_compressor_finalize, |
||||
.free = bbstreamer_zstd_compressor_free |
||||
}; |
||||
|
||||
static void bbstreamer_zstd_decompressor_content(bbstreamer *streamer, |
||||
bbstreamer_member *member, |
||||
const char *data, int len, |
||||
bbstreamer_archive_context context); |
||||
static void bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer); |
||||
static void bbstreamer_zstd_decompressor_free(bbstreamer *streamer); |
||||
|
||||
const bbstreamer_ops bbstreamer_zstd_decompressor_ops = { |
||||
.content = bbstreamer_zstd_decompressor_content, |
||||
.finalize = bbstreamer_zstd_decompressor_finalize, |
||||
.free = bbstreamer_zstd_decompressor_free |
||||
}; |
||||
#endif |
||||
|
||||
/*
|
||||
* Create a new base backup streamer that performs zstd compression of tar |
||||
* blocks. |
||||
*/ |
||||
bbstreamer * |
||||
bbstreamer_zstd_compressor_new(bbstreamer *next, int compresslevel) |
||||
{ |
||||
#ifdef HAVE_LIBZSTD |
||||
bbstreamer_zstd_frame *streamer; |
||||
|
||||
Assert(next != NULL); |
||||
|
||||
streamer = palloc0(sizeof(bbstreamer_zstd_frame)); |
||||
|
||||
*((const bbstreamer_ops **) &streamer->base.bbs_ops) = |
||||
&bbstreamer_zstd_compressor_ops; |
||||
|
||||
streamer->base.bbs_next = next; |
||||
initStringInfo(&streamer->base.bbs_buffer); |
||||
enlargeStringInfo(&streamer->base.bbs_buffer, ZSTD_DStreamOutSize()); |
||||
|
||||
streamer->cctx = ZSTD_createCCtx(); |
||||
if (!streamer->cctx) |
||||
pg_log_error("could not create zstd compression context"); |
||||
|
||||
/* Initialize stream compression preferences */ |
||||
ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_compressionLevel, |
||||
compresslevel); |
||||
|
||||
/* Initialize the ZSTD output buffer. */ |
||||
streamer->zstd_outBuf.dst = streamer->base.bbs_buffer.data; |
||||
streamer->zstd_outBuf.size = streamer->base.bbs_buffer.maxlen; |
||||
streamer->zstd_outBuf.pos = 0; |
||||
|
||||
return &streamer->base; |
||||
#else |
||||
pg_log_error("this build does not support zstd compression"); |
||||
exit(1); |
||||
#endif |
||||
} |
||||
|
||||
#ifdef HAVE_LIBZSTD |
||||
/*
|
||||
* Compress the input data to output buffer. |
||||
* |
||||
* Find out the compression bound based on input data length for each |
||||
* invocation to make sure that output buffer has enough capacity to |
||||
* accommodate the compressed data. In case if the output buffer |
||||
* capacity falls short of compression bound then forward the content |
||||
* of output buffer to next streamer and empty the buffer. |
||||
*/ |
||||
static void |
||||
bbstreamer_zstd_compressor_content(bbstreamer *streamer, |
||||
bbstreamer_member *member, |
||||
const char *data, int len, |
||||
bbstreamer_archive_context context) |
||||
{ |
||||
bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer; |
||||
ZSTD_inBuffer inBuf = {data, len, 0}; |
||||
|
||||
while (inBuf.pos < inBuf.size) |
||||
{ |
||||
size_t yet_to_flush; |
||||
size_t max_needed = ZSTD_compressBound(inBuf.size - inBuf.pos); |
||||
|
||||
/*
|
||||
* If the output buffer is not left with enough space, send the |
||||
* compressed bytes to the next streamer, and empty the buffer. |
||||
*/ |
||||
if (mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos < |
||||
max_needed) |
||||
{ |
||||
bbstreamer_content(mystreamer->base.bbs_next, member, |
||||
mystreamer->zstd_outBuf.dst, |
||||
mystreamer->zstd_outBuf.pos, |
||||
context); |
||||
|
||||
/* Reset the ZSTD output buffer. */ |
||||
mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data; |
||||
mystreamer->zstd_outBuf.size = mystreamer->base.bbs_buffer.maxlen; |
||||
mystreamer->zstd_outBuf.pos = 0; |
||||
} |
||||
|
||||
yet_to_flush = |
||||
ZSTD_compressStream2(mystreamer->cctx, &mystreamer->zstd_outBuf, |
||||
&inBuf, ZSTD_e_continue); |
||||
|
||||
if (ZSTD_isError(yet_to_flush)) |
||||
pg_log_error("could not compress data: %s", |
||||
ZSTD_getErrorName(yet_to_flush)); |
||||
} |
||||
} |
||||
|
||||
/*
|
||||
* End-of-stream processing. |
||||
*/ |
||||
static void |
||||
bbstreamer_zstd_compressor_finalize(bbstreamer *streamer) |
||||
{ |
||||
bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer; |
||||
size_t yet_to_flush; |
||||
|
||||
do |
||||
{ |
||||
ZSTD_inBuffer in = {NULL, 0, 0}; |
||||
size_t max_needed = ZSTD_compressBound(0); |
||||
|
||||
/*
|
||||
* If the output buffer is not left with enough space, send the |
||||
* compressed bytes to the next streamer, and empty the buffer. |
||||
*/ |
||||
if (mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos < |
||||
max_needed) |
||||
{ |
||||
bbstreamer_content(mystreamer->base.bbs_next, NULL, |
||||
mystreamer->zstd_outBuf.dst, |
||||
mystreamer->zstd_outBuf.pos, |
||||
BBSTREAMER_UNKNOWN); |
||||
|
||||
/* Reset the ZSTD output buffer. */ |
||||
mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data; |
||||
mystreamer->zstd_outBuf.size = mystreamer->base.bbs_buffer.maxlen; |
||||
mystreamer->zstd_outBuf.pos = 0; |
||||
} |
||||
|
||||
yet_to_flush = ZSTD_compressStream2(mystreamer->cctx, |
||||
&mystreamer->zstd_outBuf, |
||||
&in, ZSTD_e_end); |
||||
|
||||
if (ZSTD_isError(yet_to_flush)) |
||||
pg_log_error("could not compress data: %s", |
||||
ZSTD_getErrorName(yet_to_flush)); |
||||
|
||||
} while (yet_to_flush > 0); |
||||
|
||||
/* Make sure to pass any remaining bytes to the next streamer. */ |
||||
if (mystreamer->zstd_outBuf.pos > 0) |
||||
bbstreamer_content(mystreamer->base.bbs_next, NULL, |
||||
mystreamer->zstd_outBuf.dst, |
||||
mystreamer->zstd_outBuf.pos, |
||||
BBSTREAMER_UNKNOWN); |
||||
|
||||
bbstreamer_finalize(mystreamer->base.bbs_next); |
||||
} |
||||
|
||||
/*
|
||||
* Free memory. |
||||
*/ |
||||
static void |
||||
bbstreamer_zstd_compressor_free(bbstreamer *streamer) |
||||
{ |
||||
bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer; |
||||
|
||||
bbstreamer_free(streamer->bbs_next); |
||||
ZSTD_freeCCtx(mystreamer->cctx); |
||||
pfree(streamer->bbs_buffer.data); |
||||
pfree(streamer); |
||||
} |
||||
#endif |
||||
|
||||
/*
|
||||
* Create a new base backup streamer that performs decompression of zstd |
||||
* compressed blocks. |
||||
*/ |
||||
bbstreamer * |
||||
bbstreamer_zstd_decompressor_new(bbstreamer *next) |
||||
{ |
||||
#ifdef HAVE_LIBZSTD |
||||
bbstreamer_zstd_frame *streamer; |
||||
|
||||
Assert(next != NULL); |
||||
|
||||
streamer = palloc0(sizeof(bbstreamer_zstd_frame)); |
||||
*((const bbstreamer_ops **) &streamer->base.bbs_ops) = |
||||
&bbstreamer_zstd_decompressor_ops; |
||||
|
||||
streamer->base.bbs_next = next; |
||||
initStringInfo(&streamer->base.bbs_buffer); |
||||
enlargeStringInfo(&streamer->base.bbs_buffer, ZSTD_DStreamOutSize()); |
||||
|
||||
streamer->dctx = ZSTD_createDCtx(); |
||||
if (!streamer->dctx) |
||||
{ |
||||
pg_log_error("could not create zstd decompression context"); |
||||
exit(1); |
||||
} |
||||
|
||||
/* Initialize the ZSTD output buffer. */ |
||||
streamer->zstd_outBuf.dst = streamer->base.bbs_buffer.data; |
||||
streamer->zstd_outBuf.size = streamer->base.bbs_buffer.maxlen; |
||||
streamer->zstd_outBuf.pos = 0; |
||||
|
||||
return &streamer->base; |
||||
#else |
||||
pg_log_error("this build does not support compression"); |
||||
exit(1); |
||||
#endif |
||||
} |
||||
|
||||
#ifdef HAVE_LIBZSTD |
||||
/*
|
||||
* Decompress the input data to output buffer until we run out of input |
||||
* data. Each time the output buffer is full, pass on the decompressed data |
||||
* to the next streamer. |
||||
*/ |
||||
static void |
||||
bbstreamer_zstd_decompressor_content(bbstreamer *streamer, |
||||
bbstreamer_member *member, |
||||
const char *data, int len, |
||||
bbstreamer_archive_context context) |
||||
{ |
||||
bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer; |
||||
ZSTD_inBuffer inBuf = {data, len, 0}; |
||||
|
||||
while (inBuf.pos < inBuf.size) |
||||
{ |
||||
size_t ret; |
||||
|
||||
/*
|
||||
* If output buffer is full then forward the content to next streamer |
||||
* and update the output buffer. |
||||
*/ |
||||
if (mystreamer->zstd_outBuf.pos >= mystreamer->zstd_outBuf.size) |
||||
{ |
||||
bbstreamer_content(mystreamer->base.bbs_next, member, |
||||
mystreamer->zstd_outBuf.dst, |
||||
mystreamer->zstd_outBuf.pos, |
||||
context); |
||||
|
||||
/* Reset the ZSTD output buffer. */ |
||||
mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data; |
||||
mystreamer->zstd_outBuf.size = mystreamer->base.bbs_buffer.maxlen; |
||||
mystreamer->zstd_outBuf.pos = 0; |
||||
} |
||||
|
||||
ret = ZSTD_decompressStream(mystreamer->dctx, |
||||
&mystreamer->zstd_outBuf, &inBuf); |
||||
|
||||
if (ZSTD_isError(ret)) |
||||
pg_log_error("could not decompress data: %s", ZSTD_getErrorName(ret)); |
||||
} |
||||
} |
||||
|
||||
/*
|
||||
* End-of-stream processing. |
||||
*/ |
||||
static void |
||||
bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer) |
||||
{ |
||||
bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer; |
||||
|
||||
/*
|
||||
* End of the stream, if there is some pending data in output buffers then |
||||
* we must forward it to next streamer. |
||||
*/ |
||||
if (mystreamer->zstd_outBuf.pos > 0) |
||||
bbstreamer_content(mystreamer->base.bbs_next, NULL, |
||||
mystreamer->base.bbs_buffer.data, |
||||
mystreamer->base.bbs_buffer.maxlen, |
||||
BBSTREAMER_UNKNOWN); |
||||
|
||||
bbstreamer_finalize(mystreamer->base.bbs_next); |
||||
} |
||||
|
||||
/*
|
||||
* Free memory. |
||||
*/ |
||||
static void |
||||
bbstreamer_zstd_decompressor_free(bbstreamer *streamer) |
||||
{ |
||||
bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer; |
||||
|
||||
bbstreamer_free(streamer->bbs_next); |
||||
ZSTD_freeDCtx(mystreamer->dctx); |
||||
pfree(streamer->bbs_buffer.data); |
||||
pfree(streamer); |
||||
} |
||||
#endif |
||||
Loading…
Reference in new issue