mirror of https://github.com/postgres/postgres
Switch pg_dump to use the Compression API, implemented by bf9aa490db.
The CompressFileHandle replaces the cfp* family of functions with a
struct of callbacks for accessing (compressed) files. This allows adding
new compression methods simply by introducing a new struct instance with
appropriate implementation of the callbacks.
Archives compressed using custom compression methods store an identifier
of the compression algorithm in their header instead of the compression
level. The header version is bumped.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Rachel Heaton, Justin Pryzby, Tomas Vondra
Discussion: https://postgr.es/m/faUNEOpts9vunEaLnmxmG-DldLSg_ql137OC3JYDmgrOMHm1RvvWY2IdBkv_CRxm5spCCb_OmKNk2T03TMm0fBEWveFF9wA1WizPuAgB7Ss%3D%40protonmail.com
pull/131/head
Tomas Vondra
2 years ago
parent
739f1d6218
commit
e9960732a9
@ -0,0 +1,401 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* compress_gzip.c |
||||
* Routines for archivers to read or write a gzip compressed data stream. |
||||
* |
||||
* Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group |
||||
* Portions Copyright (c) 1994, Regents of the University of California |
||||
* |
||||
* IDENTIFICATION |
||||
* src/bin/pg_dump/compress_gzip.c |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
#include "postgres_fe.h" |
||||
#include <unistd.h> |
||||
|
||||
#include "compress_gzip.h" |
||||
#include "pg_backup_utils.h" |
||||
|
||||
#ifdef HAVE_LIBZ |
||||
#include "zlib.h" |
||||
|
||||
/*----------------------
|
||||
* Compressor API |
||||
*---------------------- |
||||
*/ |
||||
typedef struct GzipCompressorState |
||||
{ |
||||
z_streamp zp; |
||||
|
||||
void *outbuf; |
||||
size_t outsize; |
||||
} GzipCompressorState; |
||||
|
||||
/* Private routines that support gzip compressed data I/O */ |
||||
static void |
||||
DeflateCompressorGzip(ArchiveHandle *AH, CompressorState *cs, bool flush) |
||||
{ |
||||
GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private_data; |
||||
z_streamp zp = gzipcs->zp; |
||||
void *out = gzipcs->outbuf; |
||||
int res = Z_OK; |
||||
|
||||
while (gzipcs->zp->avail_in != 0 || flush) |
||||
{ |
||||
res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH); |
||||
if (res == Z_STREAM_ERROR) |
||||
pg_fatal("could not compress data: %s", zp->msg); |
||||
if ((flush && (zp->avail_out < gzipcs->outsize)) |
||||
|| (zp->avail_out == 0) |
||||
|| (zp->avail_in != 0) |
||||
) |
||||
{ |
||||
/*
|
||||
* Extra paranoia: avoid zero-length chunks, since a zero length |
||||
* chunk is the EOF marker in the custom format. This should never |
||||
* happen but ... |
||||
*/ |
||||
if (zp->avail_out < gzipcs->outsize) |
||||
{ |
||||
/*
|
||||
* Any write function should do its own error checking but to |
||||
* make sure we do a check here as well ... |
||||
*/ |
||||
size_t len = gzipcs->outsize - zp->avail_out; |
||||
|
||||
cs->writeF(AH, (char *) out, len); |
||||
} |
||||
zp->next_out = out; |
||||
zp->avail_out = gzipcs->outsize; |
||||
} |
||||
|
||||
if (res == Z_STREAM_END) |
||||
break; |
||||
} |
||||
} |
||||
|
||||
static void |
||||
EndCompressorGzip(ArchiveHandle *AH, CompressorState *cs) |
||||
{ |
||||
GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private_data; |
||||
z_streamp zp; |
||||
|
||||
if (gzipcs->zp) |
||||
{ |
||||
zp = gzipcs->zp; |
||||
zp->next_in = NULL; |
||||
zp->avail_in = 0; |
||||
|
||||
/* Flush any remaining data from zlib buffer */ |
||||
DeflateCompressorGzip(AH, cs, true); |
||||
|
||||
if (deflateEnd(zp) != Z_OK) |
||||
pg_fatal("could not close compression stream: %s", zp->msg); |
||||
|
||||
pg_free(gzipcs->outbuf); |
||||
pg_free(gzipcs->zp); |
||||
} |
||||
|
||||
pg_free(gzipcs); |
||||
cs->private_data = NULL; |
||||
} |
||||
|
||||
static void |
||||
WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs, |
||||
const void *data, size_t dLen) |
||||
{ |
||||
GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private_data; |
||||
z_streamp zp; |
||||
|
||||
if (!gzipcs->zp) |
||||
{ |
||||
zp = gzipcs->zp = (z_streamp) pg_malloc(sizeof(z_stream)); |
||||
zp->zalloc = Z_NULL; |
||||
zp->zfree = Z_NULL; |
||||
zp->opaque = Z_NULL; |
||||
|
||||
/*
|
||||
* outsize is the buffer size we tell zlib it can output to. We |
||||
* actually allocate one extra byte because some routines want to |
||||
* append a trailing zero byte to the zlib output. |
||||
*/ |
||||
gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1); |
||||
gzipcs->outsize = ZLIB_OUT_SIZE; |
||||
|
||||
/*
|
||||
* A level of zero simply copies the input one block at the time. This |
||||
* is probably not what the user wanted when calling this interface. |
||||
*/ |
||||
if (cs->compression_spec.level == 0) |
||||
pg_fatal("requested to compress the archive yet no level was specified"); |
||||
|
||||
if (deflateInit(zp, cs->compression_spec.level) != Z_OK) |
||||
pg_fatal("could not initialize compression library: %s", zp->msg); |
||||
|
||||
/* Just be paranoid - maybe End is called after Start, with no Write */ |
||||
zp->next_out = gzipcs->outbuf; |
||||
zp->avail_out = gzipcs->outsize; |
||||
} |
||||
|
||||
gzipcs->zp->next_in = (void *) unconstify(void *, data); |
||||
gzipcs->zp->avail_in = dLen; |
||||
DeflateCompressorGzip(AH, cs, false); |
||||
} |
||||
|
||||
static void |
||||
ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs) |
||||
{ |
||||
z_streamp zp; |
||||
char *out; |
||||
int res = Z_OK; |
||||
size_t cnt; |
||||
char *buf; |
||||
size_t buflen; |
||||
|
||||
zp = (z_streamp) pg_malloc(sizeof(z_stream)); |
||||
zp->zalloc = Z_NULL; |
||||
zp->zfree = Z_NULL; |
||||
zp->opaque = Z_NULL; |
||||
|
||||
buf = pg_malloc(ZLIB_IN_SIZE); |
||||
buflen = ZLIB_IN_SIZE; |
||||
|
||||
out = pg_malloc(ZLIB_OUT_SIZE + 1); |
||||
|
||||
if (inflateInit(zp) != Z_OK) |
||||
pg_fatal("could not initialize compression library: %s", |
||||
zp->msg); |
||||
|
||||
/* no minimal chunk size for zlib */ |
||||
while ((cnt = cs->readF(AH, &buf, &buflen))) |
||||
{ |
||||
zp->next_in = (void *) buf; |
||||
zp->avail_in = cnt; |
||||
|
||||
while (zp->avail_in > 0) |
||||
{ |
||||
zp->next_out = (void *) out; |
||||
zp->avail_out = ZLIB_OUT_SIZE; |
||||
|
||||
res = inflate(zp, 0); |
||||
if (res != Z_OK && res != Z_STREAM_END) |
||||
pg_fatal("could not uncompress data: %s", zp->msg); |
||||
|
||||
out[ZLIB_OUT_SIZE - zp->avail_out] = '\0'; |
||||
ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH); |
||||
} |
||||
} |
||||
|
||||
zp->next_in = NULL; |
||||
zp->avail_in = 0; |
||||
while (res != Z_STREAM_END) |
||||
{ |
||||
zp->next_out = (void *) out; |
||||
zp->avail_out = ZLIB_OUT_SIZE; |
||||
res = inflate(zp, 0); |
||||
if (res != Z_OK && res != Z_STREAM_END) |
||||
pg_fatal("could not uncompress data: %s", zp->msg); |
||||
|
||||
out[ZLIB_OUT_SIZE - zp->avail_out] = '\0'; |
||||
ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH); |
||||
} |
||||
|
||||
if (inflateEnd(zp) != Z_OK) |
||||
pg_fatal("could not close compression library: %s", zp->msg); |
||||
|
||||
free(buf); |
||||
free(out); |
||||
free(zp); |
||||
} |
||||
|
||||
/* Public routines that support gzip compressed data I/O */ |
||||
void |
||||
InitCompressorGzip(CompressorState *cs, |
||||
const pg_compress_specification compression_spec) |
||||
{ |
||||
GzipCompressorState *gzipcs; |
||||
|
||||
cs->readData = ReadDataFromArchiveGzip; |
||||
cs->writeData = WriteDataToArchiveGzip; |
||||
cs->end = EndCompressorGzip; |
||||
|
||||
cs->compression_spec = compression_spec; |
||||
|
||||
gzipcs = (GzipCompressorState *) pg_malloc0(sizeof(GzipCompressorState)); |
||||
|
||||
cs->private_data = gzipcs; |
||||
} |
||||
|
||||
|
||||
/*----------------------
|
||||
* Compress File API |
||||
*---------------------- |
||||
*/ |
||||
|
||||
static size_t |
||||
Gzip_read(void *ptr, size_t size, CompressFileHandle *CFH) |
||||
{ |
||||
gzFile gzfp = (gzFile) CFH->private_data; |
||||
size_t ret; |
||||
|
||||
ret = gzread(gzfp, ptr, size); |
||||
if (ret != size && !gzeof(gzfp)) |
||||
{ |
||||
int errnum; |
||||
const char *errmsg = gzerror(gzfp, &errnum); |
||||
|
||||
pg_fatal("could not read from input file: %s", |
||||
errnum == Z_ERRNO ? strerror(errno) : errmsg); |
||||
} |
||||
|
||||
return ret; |
||||
} |
||||
|
||||
static size_t |
||||
Gzip_write(const void *ptr, size_t size, CompressFileHandle *CFH) |
||||
{ |
||||
gzFile gzfp = (gzFile) CFH->private_data; |
||||
|
||||
return gzwrite(gzfp, ptr, size); |
||||
} |
||||
|
||||
static int |
||||
Gzip_getc(CompressFileHandle *CFH) |
||||
{ |
||||
gzFile gzfp = (gzFile) CFH->private_data; |
||||
int ret; |
||||
|
||||
errno = 0; |
||||
ret = gzgetc(gzfp); |
||||
if (ret == EOF) |
||||
{ |
||||
if (!gzeof(gzfp)) |
||||
pg_fatal("could not read from input file: %s", strerror(errno)); |
||||
else |
||||
pg_fatal("could not read from input file: end of file"); |
||||
} |
||||
|
||||
return ret; |
||||
} |
||||
|
||||
static char * |
||||
Gzip_gets(char *ptr, int size, CompressFileHandle *CFH) |
||||
{ |
||||
gzFile gzfp = (gzFile) CFH->private_data; |
||||
|
||||
return gzgets(gzfp, ptr, size); |
||||
} |
||||
|
||||
static int |
||||
Gzip_close(CompressFileHandle *CFH) |
||||
{ |
||||
gzFile gzfp = (gzFile) CFH->private_data; |
||||
|
||||
CFH->private_data = NULL; |
||||
|
||||
return gzclose(gzfp); |
||||
} |
||||
|
||||
static int |
||||
Gzip_eof(CompressFileHandle *CFH) |
||||
{ |
||||
gzFile gzfp = (gzFile) CFH->private_data; |
||||
|
||||
return gzeof(gzfp); |
||||
} |
||||
|
||||
static const char * |
||||
Gzip_get_error(CompressFileHandle *CFH) |
||||
{ |
||||
gzFile gzfp = (gzFile) CFH->private_data; |
||||
const char *errmsg; |
||||
int errnum; |
||||
|
||||
errmsg = gzerror(gzfp, &errnum); |
||||
if (errnum == Z_ERRNO) |
||||
errmsg = strerror(errno); |
||||
|
||||
return errmsg; |
||||
} |
||||
|
||||
static int |
||||
Gzip_open(const char *path, int fd, const char *mode, CompressFileHandle *CFH) |
||||
{ |
||||
gzFile gzfp; |
||||
char mode_compression[32]; |
||||
|
||||
if (CFH->compression_spec.level != Z_DEFAULT_COMPRESSION) |
||||
{ |
||||
/*
|
||||
* user has specified a compression level, so tell zlib to use it |
||||
*/ |
||||
snprintf(mode_compression, sizeof(mode_compression), "%s%d", |
||||
mode, CFH->compression_spec.level); |
||||
} |
||||
else |
||||
strcpy(mode_compression, mode); |
||||
|
||||
if (fd >= 0) |
||||
gzfp = gzdopen(dup(fd), mode_compression); |
||||
else |
||||
gzfp = gzopen(path, mode_compression); |
||||
|
||||
if (gzfp == NULL) |
||||
return 1; |
||||
|
||||
CFH->private_data = gzfp; |
||||
|
||||
return 0; |
||||
} |
||||
|
||||
static int |
||||
Gzip_open_write(const char *path, const char *mode, CompressFileHandle *CFH) |
||||
{ |
||||
char *fname; |
||||
int ret; |
||||
int save_errno; |
||||
|
||||
fname = psprintf("%s.gz", path); |
||||
ret = CFH->open_func(fname, -1, mode, CFH); |
||||
|
||||
save_errno = errno; |
||||
pg_free(fname); |
||||
errno = save_errno; |
||||
|
||||
return ret; |
||||
} |
||||
|
||||
void |
||||
InitCompressFileHandleGzip(CompressFileHandle *CFH, |
||||
const pg_compress_specification compression_spec) |
||||
{ |
||||
CFH->open_func = Gzip_open; |
||||
CFH->open_write_func = Gzip_open_write; |
||||
CFH->read_func = Gzip_read; |
||||
CFH->write_func = Gzip_write; |
||||
CFH->gets_func = Gzip_gets; |
||||
CFH->getc_func = Gzip_getc; |
||||
CFH->close_func = Gzip_close; |
||||
CFH->eof_func = Gzip_eof; |
||||
CFH->get_error_func = Gzip_get_error; |
||||
|
||||
CFH->compression_spec = compression_spec; |
||||
|
||||
CFH->private_data = NULL; |
||||
} |
||||
#else /* HAVE_LIBZ */ |
||||
void |
||||
InitCompressorGzip(CompressorState *cs, |
||||
const pg_compress_specification compression_spec) |
||||
{ |
||||
pg_fatal("this build does not support compression with %s", "gzip"); |
||||
} |
||||
|
||||
void |
||||
InitCompressFileHandleGzip(CompressFileHandle *CFH, |
||||
const pg_compress_specification compression_spec) |
||||
{ |
||||
pg_fatal("this build does not support compression with %s", "gzip"); |
||||
} |
||||
#endif /* HAVE_LIBZ */ |
||||
@ -0,0 +1,24 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* compress_gzip.h |
||||
* GZIP interface to compress_io.c routines |
||||
* |
||||
* Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group |
||||
* Portions Copyright (c) 1994, Regents of the University of California |
||||
* |
||||
* IDENTIFICATION |
||||
* src/bin/pg_dump/compress_gzip.h |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
#ifndef _COMPRESS_GZIP_H_ |
||||
#define _COMPRESS_GZIP_H_ |
||||
|
||||
#include "compress_io.h" |
||||
|
||||
extern void InitCompressorGzip(CompressorState *cs, |
||||
const pg_compress_specification compression_spec); |
||||
extern void InitCompressFileHandleGzip(CompressFileHandle *CFH, |
||||
const pg_compress_specification compression_spec); |
||||
|
||||
#endif /* _COMPRESS_GZIP_H_ */ |
||||
@ -0,0 +1,206 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* compress_none.c |
||||
* Routines for archivers to read or write an uncompressed stream. |
||||
* |
||||
* Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group |
||||
* Portions Copyright (c) 1994, Regents of the University of California |
||||
* |
||||
* IDENTIFICATION |
||||
* src/bin/pg_dump/compress_none.c |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
#include "postgres_fe.h" |
||||
#include <unistd.h> |
||||
|
||||
#include "compress_none.h" |
||||
#include "pg_backup_utils.h" |
||||
|
||||
/*----------------------
|
||||
* Compressor API |
||||
*---------------------- |
||||
*/ |
||||
|
||||
/*
|
||||
* Private routines |
||||
*/ |
||||
|
||||
static void |
||||
ReadDataFromArchiveNone(ArchiveHandle *AH, CompressorState *cs) |
||||
{ |
||||
size_t cnt; |
||||
char *buf; |
||||
size_t buflen; |
||||
|
||||
buf = pg_malloc(ZLIB_OUT_SIZE); |
||||
buflen = ZLIB_OUT_SIZE; |
||||
|
||||
while ((cnt = cs->readF(AH, &buf, &buflen))) |
||||
{ |
||||
ahwrite(buf, 1, cnt, AH); |
||||
} |
||||
|
||||
free(buf); |
||||
} |
||||
|
||||
|
||||
static void |
||||
WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, |
||||
const void *data, size_t dLen) |
||||
{ |
||||
cs->writeF(AH, data, dLen); |
||||
} |
||||
|
||||
static void |
||||
EndCompressorNone(ArchiveHandle *AH, CompressorState *cs) |
||||
{ |
||||
/* no op */ |
||||
} |
||||
|
||||
/*
|
||||
* Public interface |
||||
*/ |
||||
|
||||
void |
||||
InitCompressorNone(CompressorState *cs, |
||||
const pg_compress_specification compression_spec) |
||||
{ |
||||
cs->readData = ReadDataFromArchiveNone; |
||||
cs->writeData = WriteDataToArchiveNone; |
||||
cs->end = EndCompressorNone; |
||||
|
||||
cs->compression_spec = compression_spec; |
||||
} |
||||
|
||||
|
||||
/*----------------------
|
||||
* Compress File API |
||||
*---------------------- |
||||
*/ |
||||
|
||||
/*
|
||||
* Private routines |
||||
*/ |
||||
|
||||
static size_t |
||||
read_none(void *ptr, size_t size, CompressFileHandle *CFH) |
||||
{ |
||||
FILE *fp = (FILE *) CFH->private_data; |
||||
size_t ret; |
||||
|
||||
if (size == 0) |
||||
return 0; |
||||
|
||||
ret = fread(ptr, 1, size, fp); |
||||
if (ret != size && !feof(fp)) |
||||
pg_fatal("could not read from input file: %s", |
||||
strerror(errno)); |
||||
|
||||
return ret; |
||||
} |
||||
|
||||
static size_t |
||||
write_none(const void *ptr, size_t size, CompressFileHandle *CFH) |
||||
{ |
||||
return fwrite(ptr, 1, size, (FILE *) CFH->private_data); |
||||
} |
||||
|
||||
static const char * |
||||
get_error_none(CompressFileHandle *CFH) |
||||
{ |
||||
return strerror(errno); |
||||
} |
||||
|
||||
static char * |
||||
gets_none(char *ptr, int size, CompressFileHandle *CFH) |
||||
{ |
||||
return fgets(ptr, size, (FILE *) CFH->private_data); |
||||
} |
||||
|
||||
static int |
||||
getc_none(CompressFileHandle *CFH) |
||||
{ |
||||
FILE *fp = (FILE *) CFH->private_data; |
||||
int ret; |
||||
|
||||
ret = fgetc(fp); |
||||
if (ret == EOF) |
||||
{ |
||||
if (!feof(fp)) |
||||
pg_fatal("could not read from input file: %s", strerror(errno)); |
||||
else |
||||
pg_fatal("could not read from input file: end of file"); |
||||
} |
||||
|
||||
return ret; |
||||
} |
||||
|
||||
static int |
||||
close_none(CompressFileHandle *CFH) |
||||
{ |
||||
FILE *fp = (FILE *) CFH->private_data; |
||||
int ret = 0; |
||||
|
||||
CFH->private_data = NULL; |
||||
|
||||
if (fp) |
||||
ret = fclose(fp); |
||||
|
||||
return ret; |
||||
} |
||||
|
||||
static int |
||||
eof_none(CompressFileHandle *CFH) |
||||
{ |
||||
return feof((FILE *) CFH->private_data); |
||||
} |
||||
|
||||
static int |
||||
open_none(const char *path, int fd, const char *mode, CompressFileHandle *CFH) |
||||
{ |
||||
Assert(CFH->private_data == NULL); |
||||
|
||||
if (fd >= 0) |
||||
CFH->private_data = fdopen(dup(fd), mode); |
||||
else |
||||
CFH->private_data = fopen(path, mode); |
||||
|
||||
if (CFH->private_data == NULL) |
||||
return 1; |
||||
|
||||
return 0; |
||||
} |
||||
|
||||
static int |
||||
open_write_none(const char *path, const char *mode, CompressFileHandle *CFH) |
||||
{ |
||||
Assert(CFH->private_data == NULL); |
||||
|
||||
CFH->private_data = fopen(path, mode); |
||||
if (CFH->private_data == NULL) |
||||
return 1; |
||||
|
||||
return 0; |
||||
} |
||||
|
||||
/*
|
||||
* Public interface |
||||
*/ |
||||
|
||||
void |
||||
InitCompressFileHandleNone(CompressFileHandle *CFH, |
||||
const pg_compress_specification compression_spec) |
||||
{ |
||||
CFH->open_func = open_none; |
||||
CFH->open_write_func = open_write_none; |
||||
CFH->read_func = read_none; |
||||
CFH->write_func = write_none; |
||||
CFH->gets_func = gets_none; |
||||
CFH->getc_func = getc_none; |
||||
CFH->close_func = close_none; |
||||
CFH->eof_func = eof_none; |
||||
CFH->get_error_func = get_error_none; |
||||
|
||||
CFH->private_data = NULL; |
||||
} |
||||
@ -0,0 +1,24 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* compress_none.h |
||||
* Uncompressed interface to compress_io.c routines |
||||
* |
||||
* Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group |
||||
* Portions Copyright (c) 1994, Regents of the University of California |
||||
* |
||||
* IDENTIFICATION |
||||
* src/bin/pg_dump/compress_none.h |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
#ifndef _COMPRESS_NONE_H_ |
||||
#define _COMPRESS_NONE_H_ |
||||
|
||||
#include "compress_io.h" |
||||
|
||||
extern void InitCompressorNone(CompressorState *cs, |
||||
const pg_compress_specification compression_spec); |
||||
extern void InitCompressFileHandleNone(CompressFileHandle *CFH, |
||||
const pg_compress_specification compression_spec); |
||||
|
||||
#endif /* _COMPRESS_NONE_H_ */ |
||||
Loading…
Reference in new issue