mirror of https://github.com/postgres/postgres
There is now a per-column COMPRESSION option which can be set to pglz (the default, and the only option in up until now) or lz4. Or, if you like, you can set the new default_toast_compression GUC to lz4, and then that will be the default for new table columns for which no value is specified. We don't have lz4 support in the PostgreSQL code, so to use lz4 compression, PostgreSQL must be built --with-lz4. In general, TOAST compression means compression of individual column values, not the whole tuple, and those values can either be compressed inline within the tuple or compressed and then stored externally in the TOAST table, so those properties also apply to this feature. Prior to this commit, a TOAST pointer has two unused bits as part of the va_extsize field, and a compessed datum has two unused bits as part of the va_rawsize field. These bits are unused because the length of a varlena is limited to 1GB; we now use them to indicate the compression type that was used. This means we only have bit space for 2 more built-in compresison types, but we could work around that problem, if necessary, by introducing a new vartag_external value for any further types we end up wanting to add. Hopefully, it won't be too important to offer a wide selection of algorithms here, since each one we add not only takes more coding but also adds a build dependency for every packager. Nevertheless, it seems worth doing at least this much, because LZ4 gets better compression than PGLZ with less CPU usage. It's possible for LZ4-compressed datums to leak into composite type values stored on disk, just as it is for PGLZ. It's also possible for LZ4-compressed attributes to be copied into a different table via SQL commands such as CREATE TABLE AS or INSERT .. SELECT. It would be expensive to force such values to be decompressed, so PostgreSQL has never done so. For the same reasons, we also don't force recompression of already-compressed values even if the target table prefers a different compression method than was used for the source data. These architectural decisions are perhaps arguable but revisiting them is well beyond the scope of what seemed possible to do as part of this project. However, it's relatively cheap to recompress as part of VACUUM FULL or CLUSTER, so this commit adjusts those commands to do so, if the configured compression method of the table happens not to match what was used for some column value stored therein. Dilip Kumar. The original patches on which this work was based were written by Ildus Kurbangaliev, and those were patches were based on even earlier work by Nikita Glukhov, but the design has since changed very substantially, since allow a potentially large number of compression methods that could be added and dropped on a running system proved too problematic given some of the architectural issues mentioned above; the choice of which specific compression method to add first is now different; and a lot of the code has been heavily refactored. More recently, Justin Przyby helped quite a bit with testing and reviewing and this version also includes some code contributions from him. Other design input and review from Tomas Vondra, Álvaro Herrera, Andres Freund, Oleg Bartunov, Alexander Korotkov, and me. Discussion: http://postgr.es/m/20170907194236.4cefce96%40wp.localdomain Discussion: http://postgr.es/m/CAFiTN-uUpX3ck%3DK0mLEk-G_kUQY%3DSNOTeqdaNRR9FMdQrHKebw%40mail.gmail.compull/64/head
Robert Haas
5 years ago
parent
e589c4890b
commit
bbe0a81db6
@ -0,0 +1,313 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* toast_compression.c |
||||
* Functions for toast compression. |
||||
* |
||||
* Copyright (c) 2021, PostgreSQL Global Development Group |
||||
* |
||||
* |
||||
* IDENTIFICATION |
||||
* src/backend/access/common/toast_compression.c |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
#include "postgres.h" |
||||
|
||||
#ifdef USE_LZ4 |
||||
#include <lz4.h> |
||||
#endif |
||||
|
||||
#include "access/detoast.h" |
||||
#include "access/toast_compression.h" |
||||
#include "common/pg_lzcompress.h" |
||||
#include "fmgr.h" |
||||
#include "utils/builtins.h" |
||||
|
||||
/* Compile-time default */ |
||||
char *default_toast_compression = DEFAULT_TOAST_COMPRESSION; |
||||
|
||||
/*
|
||||
* Compress a varlena using PGLZ. |
||||
* |
||||
* Returns the compressed varlena, or NULL if compression fails. |
||||
*/ |
||||
struct varlena * |
||||
pglz_compress_datum(const struct varlena *value) |
||||
{ |
||||
int32 valsize, |
||||
len; |
||||
struct varlena *tmp = NULL; |
||||
|
||||
valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); |
||||
|
||||
/*
|
||||
* No point in wasting a palloc cycle if value size is outside the allowed |
||||
* range for compression. |
||||
*/ |
||||
if (valsize < PGLZ_strategy_default->min_input_size || |
||||
valsize > PGLZ_strategy_default->max_input_size) |
||||
return NULL; |
||||
|
||||
/*
|
||||
* Figure out the maximum possible size of the pglz output, add the bytes |
||||
* that will be needed for varlena overhead, and allocate that amount. |
||||
*/ |
||||
tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + |
||||
VARHDRSZ_COMPRESS); |
||||
|
||||
len = pglz_compress(VARDATA_ANY(value), |
||||
valsize, |
||||
(char *) tmp + VARHDRSZ_COMPRESS, |
||||
NULL); |
||||
if (len < 0) |
||||
{ |
||||
pfree(tmp); |
||||
return NULL; |
||||
} |
||||
|
||||
SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESS); |
||||
|
||||
return tmp; |
||||
} |
||||
|
||||
/*
|
||||
* Decompress a varlena that was compressed using PGLZ. |
||||
*/ |
||||
struct varlena * |
||||
pglz_decompress_datum(const struct varlena *value) |
||||
{ |
||||
struct varlena *result; |
||||
int32 rawsize; |
||||
|
||||
/* allocate memory for the uncompressed data */ |
||||
result = (struct varlena *) palloc(VARRAWSIZE_4B_C(value) + VARHDRSZ); |
||||
|
||||
/* decompress the data */ |
||||
rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESS, |
||||
VARSIZE(value) - VARHDRSZ_COMPRESS, |
||||
VARDATA(result), |
||||
VARRAWSIZE_4B_C(value), true); |
||||
if (rawsize < 0) |
||||
ereport(ERROR, |
||||
(errcode(ERRCODE_DATA_CORRUPTED), |
||||
errmsg_internal("compressed pglz data is corrupt"))); |
||||
|
||||
SET_VARSIZE(result, rawsize + VARHDRSZ); |
||||
|
||||
return result; |
||||
} |
||||
|
||||
/*
|
||||
* Decompress part of a varlena that was compressed using PGLZ. |
||||
*/ |
||||
struct varlena * |
||||
pglz_decompress_datum_slice(const struct varlena *value, |
||||
int32 slicelength) |
||||
{ |
||||
struct varlena *result; |
||||
int32 rawsize; |
||||
|
||||
/* allocate memory for the uncompressed data */ |
||||
result = (struct varlena *) palloc(slicelength + VARHDRSZ); |
||||
|
||||
/* decompress the data */ |
||||
rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESS, |
||||
VARSIZE(value) - VARHDRSZ_COMPRESS, |
||||
VARDATA(result), |
||||
slicelength, false); |
||||
if (rawsize < 0) |
||||
ereport(ERROR, |
||||
(errcode(ERRCODE_DATA_CORRUPTED), |
||||
errmsg_internal("compressed pglz data is corrupt"))); |
||||
|
||||
SET_VARSIZE(result, rawsize + VARHDRSZ); |
||||
|
||||
return result; |
||||
} |
||||
|
||||
/*
|
||||
* Compress a varlena using LZ4. |
||||
* |
||||
* Returns the compressed varlena, or NULL if compression fails. |
||||
*/ |
||||
struct varlena * |
||||
lz4_compress_datum(const struct varlena *value) |
||||
{ |
||||
#ifndef USE_LZ4 |
||||
NO_LZ4_SUPPORT(); |
||||
#else |
||||
int32 valsize; |
||||
int32 len; |
||||
int32 max_size; |
||||
struct varlena *tmp = NULL; |
||||
|
||||
valsize = VARSIZE_ANY_EXHDR(value); |
||||
|
||||
/*
|
||||
* Figure out the maximum possible size of the LZ4 output, add the bytes |
||||
* that will be needed for varlena overhead, and allocate that amount. |
||||
*/ |
||||
max_size = LZ4_compressBound(valsize); |
||||
tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESS); |
||||
|
||||
len = LZ4_compress_default(VARDATA_ANY(value), |
||||
(char *) tmp + VARHDRSZ_COMPRESS, |
||||
valsize, max_size); |
||||
if (len <= 0) |
||||
elog(ERROR, "lz4 compression failed"); |
||||
|
||||
/* data is incompressible so just free the memory and return NULL */ |
||||
if (len > valsize) |
||||
{ |
||||
pfree(tmp); |
||||
return NULL; |
||||
} |
||||
|
||||
SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESS); |
||||
|
||||
return tmp; |
||||
#endif |
||||
} |
||||
|
||||
/*
|
||||
* Decompress a varlena that was compressed using LZ4. |
||||
*/ |
||||
struct varlena * |
||||
lz4_decompress_datum(const struct varlena *value) |
||||
{ |
||||
#ifndef USE_LZ4 |
||||
NO_LZ4_SUPPORT(); |
||||
#else |
||||
int32 rawsize; |
||||
struct varlena *result; |
||||
|
||||
/* allocate memory for the uncompressed data */ |
||||
result = (struct varlena *) palloc(VARRAWSIZE_4B_C(value) + VARHDRSZ); |
||||
|
||||
/* decompress the data */ |
||||
rawsize = LZ4_decompress_safe((char *) value + VARHDRSZ_COMPRESS, |
||||
VARDATA(result), |
||||
VARSIZE(value) - VARHDRSZ_COMPRESS, |
||||
VARRAWSIZE_4B_C(value)); |
||||
if (rawsize < 0) |
||||
ereport(ERROR, |
||||
(errcode(ERRCODE_DATA_CORRUPTED), |
||||
errmsg_internal("compressed lz4 data is corrupt"))); |
||||
|
||||
|
||||
SET_VARSIZE(result, rawsize + VARHDRSZ); |
||||
|
||||
return result; |
||||
#endif |
||||
} |
||||
|
||||
/*
|
||||
* Decompress part of a varlena that was compressed using LZ4. |
||||
*/ |
||||
struct varlena * |
||||
lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength) |
||||
{ |
||||
#ifndef USE_LZ4 |
||||
NO_LZ4_SUPPORT(); |
||||
#else |
||||
int32 rawsize; |
||||
struct varlena *result; |
||||
|
||||
/* slice decompression not supported prior to 1.8.3 */ |
||||
if (LZ4_versionNumber() < 10803) |
||||
return lz4_decompress_datum(value); |
||||
|
||||
/* allocate memory for the uncompressed data */ |
||||
result = (struct varlena *) palloc(slicelength + VARHDRSZ); |
||||
|
||||
/* decompress the data */ |
||||
rawsize = LZ4_decompress_safe_partial((char *) value + VARHDRSZ_COMPRESS, |
||||
VARDATA(result), |
||||
VARSIZE(value) - VARHDRSZ_COMPRESS, |
||||
slicelength, |
||||
slicelength); |
||||
if (rawsize < 0) |
||||
ereport(ERROR, |
||||
(errcode(ERRCODE_DATA_CORRUPTED), |
||||
errmsg_internal("compressed lz4 data is corrupt"))); |
||||
|
||||
SET_VARSIZE(result, rawsize + VARHDRSZ); |
||||
|
||||
return result; |
||||
#endif |
||||
} |
||||
|
||||
/*
|
||||
* Extract compression ID from a varlena. |
||||
* |
||||
* Returns TOAST_INVALID_COMPRESSION_ID if the varlena is not compressed. |
||||
*/ |
||||
ToastCompressionId |
||||
toast_get_compression_id(struct varlena *attr) |
||||
{ |
||||
ToastCompressionId cmid = TOAST_INVALID_COMPRESSION_ID; |
||||
|
||||
/*
|
||||
* If it is stored externally then fetch the compression method id from the |
||||
* external toast pointer. If compressed inline, fetch it from the toast |
||||
* compression header. |
||||
*/ |
||||
if (VARATT_IS_EXTERNAL_ONDISK(attr)) |
||||
{ |
||||
struct varatt_external toast_pointer; |
||||
|
||||
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); |
||||
|
||||
if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) |
||||
cmid = VARATT_EXTERNAL_GET_COMPRESSION(toast_pointer); |
||||
} |
||||
else if (VARATT_IS_COMPRESSED(attr)) |
||||
cmid = VARCOMPRESS_4B_C(attr); |
||||
|
||||
return cmid; |
||||
} |
||||
|
||||
/*
|
||||
* Validate a new value for the default_toast_compression GUC. |
||||
*/ |
||||
bool |
||||
check_default_toast_compression(char **newval, void **extra, GucSource source) |
||||
{ |
||||
if (**newval == '\0') |
||||
{ |
||||
GUC_check_errdetail("%s cannot be empty.", |
||||
"default_toast_compression"); |
||||
return false; |
||||
} |
||||
|
||||
if (strlen(*newval) >= NAMEDATALEN) |
||||
{ |
||||
GUC_check_errdetail("%s is too long (maximum %d characters).", |
||||
"default_toast_compression", NAMEDATALEN - 1); |
||||
return false; |
||||
} |
||||
|
||||
if (!CompressionMethodIsValid(CompressionNameToMethod(*newval))) |
||||
{ |
||||
/*
|
||||
* When source == PGC_S_TEST, don't throw a hard error for a |
||||
* nonexistent compression method, only a NOTICE. See comments in |
||||
* guc.h. |
||||
*/ |
||||
if (source == PGC_S_TEST) |
||||
{ |
||||
ereport(NOTICE, |
||||
(errcode(ERRCODE_UNDEFINED_OBJECT), |
||||
errmsg("compression method \"%s\" does not exist", |
||||
*newval))); |
||||
} |
||||
else |
||||
{ |
||||
GUC_check_errdetail("Compression method \"%s\" does not exist.", |
||||
*newval); |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
return true; |
||||
} |
||||
@ -0,0 +1,123 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* toast_compression.h |
||||
* Functions for toast compression. |
||||
* |
||||
* Copyright (c) 2021, PostgreSQL Global Development Group |
||||
* |
||||
* src/include/access/toast_compression.h |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
|
||||
#ifndef TOAST_COMPRESSION_H |
||||
#define TOAST_COMPRESSION_H |
||||
|
||||
#include "utils/guc.h" |
||||
|
||||
/* GUCs */ |
||||
extern char *default_toast_compression; |
||||
|
||||
/* default compression method if not specified. */ |
||||
#define DEFAULT_TOAST_COMPRESSION "pglz" |
||||
|
||||
/*
|
||||
* Built-in compression method-id. The toast compression header will store |
||||
* this in the first 2 bits of the raw length. These built-in compression |
||||
* method-id are directly mapped to the built-in compression methods. |
||||
*/ |
||||
typedef enum ToastCompressionId |
||||
{ |
||||
TOAST_PGLZ_COMPRESSION_ID = 0, |
||||
TOAST_LZ4_COMPRESSION_ID = 1, |
||||
TOAST_INVALID_COMPRESSION_ID = 2 |
||||
} ToastCompressionId; |
||||
|
||||
/*
|
||||
* Built-in compression methods. pg_attribute will store this in the |
||||
* attcompression column. |
||||
*/ |
||||
#define TOAST_PGLZ_COMPRESSION 'p' |
||||
#define TOAST_LZ4_COMPRESSION 'l' |
||||
|
||||
#define InvalidCompressionMethod '\0' |
||||
#define CompressionMethodIsValid(cm) ((bool) ((cm) != InvalidCompressionMethod)) |
||||
|
||||
#define NO_LZ4_SUPPORT() \ |
||||
ereport(ERROR, \
|
||||
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
|
||||
errmsg("unsupported LZ4 compression method"), \
|
||||
errdetail("This functionality requires the server to be built with lz4 support."), \
|
||||
errhint("You need to rebuild PostgreSQL using --with-lz4."))) |
||||
|
||||
#define IsValidCompression(cm) ((cm) != InvalidCompressionMethod) |
||||
|
||||
#define IsStorageCompressible(storage) ((storage) != TYPSTORAGE_PLAIN && \ |
||||
(storage) != TYPSTORAGE_EXTERNAL) |
||||
|
||||
/*
|
||||
* GetCompressionMethodName - Get compression method name |
||||
*/ |
||||
static inline const char * |
||||
GetCompressionMethodName(char method) |
||||
{ |
||||
switch (method) |
||||
{ |
||||
case TOAST_PGLZ_COMPRESSION: |
||||
return "pglz"; |
||||
case TOAST_LZ4_COMPRESSION: |
||||
return "lz4"; |
||||
default: |
||||
elog(ERROR, "invalid compression method %c", method); |
||||
} |
||||
} |
||||
|
||||
/*
|
||||
* CompressionNameToMethod - Get compression method from compression name |
||||
* |
||||
* Search in the available built-in methods. If the compression not found |
||||
* in the built-in methods then return InvalidCompressionMethod. |
||||
*/ |
||||
static inline char |
||||
CompressionNameToMethod(char *compression) |
||||
{ |
||||
if (strcmp(compression, "pglz") == 0) |
||||
return TOAST_PGLZ_COMPRESSION; |
||||
else if (strcmp(compression, "lz4") == 0) |
||||
{ |
||||
#ifndef USE_LZ4 |
||||
NO_LZ4_SUPPORT(); |
||||
#endif |
||||
return TOAST_LZ4_COMPRESSION; |
||||
} |
||||
|
||||
return InvalidCompressionMethod; |
||||
} |
||||
|
||||
/*
|
||||
* GetDefaultToastCompression -- get the default toast compression method |
||||
* |
||||
* This exists to hide the use of the default_toast_compression GUC variable. |
||||
*/ |
||||
static inline char |
||||
GetDefaultToastCompression(void) |
||||
{ |
||||
return CompressionNameToMethod(default_toast_compression); |
||||
} |
||||
|
||||
/* pglz compression/decompression routines */ |
||||
extern struct varlena *pglz_compress_datum(const struct varlena *value); |
||||
extern struct varlena *pglz_decompress_datum(const struct varlena *value); |
||||
extern struct varlena *pglz_decompress_datum_slice(const struct varlena *value, |
||||
int32 slicelength); |
||||
|
||||
/* lz4 compression/decompression routines */ |
||||
extern struct varlena *lz4_compress_datum(const struct varlena *value); |
||||
extern struct varlena *lz4_decompress_datum(const struct varlena *value); |
||||
extern struct varlena *lz4_decompress_datum_slice(const struct varlena *value, |
||||
int32 slicelength); |
||||
extern ToastCompressionId toast_get_compression_id(struct varlena *attr); |
||||
extern bool check_default_toast_compression(char **newval, void **extra, |
||||
GucSource source); |
||||
|
||||
#endif /* TOAST_COMPRESSION_H */ |
||||
@ -0,0 +1,347 @@ |
||||
\set HIDE_TOAST_COMPRESSION false |
||||
-- test creating table with compression method |
||||
CREATE TABLE cmdata(f1 text COMPRESSION pglz); |
||||
CREATE INDEX idx ON cmdata(f1); |
||||
INSERT INTO cmdata VALUES(repeat('1234567890', 1000)); |
||||
\d+ cmdata |
||||
Table "public.cmdata" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | text | | | | extended | pglz | | |
||||
Indexes: |
||||
"idx" btree (f1) |
||||
|
||||
CREATE TABLE cmdata1(f1 TEXT COMPRESSION lz4); |
||||
INSERT INTO cmdata1 VALUES(repeat('1234567890', 1004)); |
||||
\d+ cmdata1 |
||||
Table "public.cmdata1" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | text | | | | extended | lz4 | | |
||||
|
||||
-- verify stored compression method in the data |
||||
SELECT pg_column_compression(f1) FROM cmdata; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
(1 row) |
||||
|
||||
SELECT pg_column_compression(f1) FROM cmdata1; |
||||
pg_column_compression |
||||
----------------------- |
||||
lz4 |
||||
(1 row) |
||||
|
||||
-- decompress data slice |
||||
SELECT SUBSTR(f1, 200, 5) FROM cmdata; |
||||
substr |
||||
-------- |
||||
01234 |
||||
(1 row) |
||||
|
||||
SELECT SUBSTR(f1, 2000, 50) FROM cmdata1; |
||||
substr |
||||
---------------------------------------------------- |
||||
01234567890123456789012345678901234567890123456789 |
||||
(1 row) |
||||
|
||||
-- copy with table creation |
||||
SELECT * INTO cmmove1 FROM cmdata; |
||||
\d+ cmmove1 |
||||
Table "public.cmmove1" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | text | | | | extended | pglz | | |
||||
|
||||
SELECT pg_column_compression(f1) FROM cmmove1; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
(1 row) |
||||
|
||||
-- copy to existing table |
||||
CREATE TABLE cmmove3(f1 text COMPRESSION pglz); |
||||
INSERT INTO cmmove3 SELECT * FROM cmdata; |
||||
INSERT INTO cmmove3 SELECT * FROM cmdata1; |
||||
SELECT pg_column_compression(f1) FROM cmmove3; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
lz4 |
||||
(2 rows) |
||||
|
||||
-- test LIKE INCLUDING COMPRESSION |
||||
CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION); |
||||
\d+ cmdata2 |
||||
Table "public.cmdata2" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | text | | | | extended | lz4 | | |
||||
|
||||
DROP TABLE cmdata2; |
||||
-- try setting compression for incompressible data type |
||||
CREATE TABLE cmdata2 (f1 int COMPRESSION pglz); |
||||
ERROR: column data type integer does not support compression |
||||
-- update using datum from different table |
||||
CREATE TABLE cmmove2(f1 text COMPRESSION pglz); |
||||
INSERT INTO cmmove2 VALUES (repeat('1234567890', 1004)); |
||||
SELECT pg_column_compression(f1) FROM cmmove2; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
(1 row) |
||||
|
||||
UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1; |
||||
SELECT pg_column_compression(f1) FROM cmmove2; |
||||
pg_column_compression |
||||
----------------------- |
||||
lz4 |
||||
(1 row) |
||||
|
||||
-- test externally stored compressed data |
||||
CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS |
||||
'select array_agg(md5(g::text))::text from generate_series(1, 256) g'; |
||||
CREATE TABLE cmdata2 (f1 text COMPRESSION pglz); |
||||
INSERT INTO cmdata2 SELECT large_val() || repeat('a', 4000); |
||||
SELECT pg_column_compression(f1) FROM cmdata2; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
(1 row) |
||||
|
||||
INSERT INTO cmdata1 SELECT large_val() || repeat('a', 4000); |
||||
SELECT pg_column_compression(f1) FROM cmdata1; |
||||
pg_column_compression |
||||
----------------------- |
||||
lz4 |
||||
lz4 |
||||
(2 rows) |
||||
|
||||
SELECT SUBSTR(f1, 200, 5) FROM cmdata1; |
||||
substr |
||||
-------- |
||||
01234 |
||||
8f14e |
||||
(2 rows) |
||||
|
||||
SELECT SUBSTR(f1, 200, 5) FROM cmdata2; |
||||
substr |
||||
-------- |
||||
8f14e |
||||
(1 row) |
||||
|
||||
DROP TABLE cmdata2; |
||||
--test column type update varlena/non-varlena |
||||
CREATE TABLE cmdata2 (f1 int); |
||||
\d+ cmdata2 |
||||
Table "public.cmdata2" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- |
||||
f1 | integer | | | | plain | | | |
||||
|
||||
ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar; |
||||
\d+ cmdata2 |
||||
Table "public.cmdata2" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | character varying | | | | extended | pglz | | |
||||
|
||||
ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE int USING f1::integer; |
||||
\d+ cmdata2 |
||||
Table "public.cmdata2" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- |
||||
f1 | integer | | | | plain | | | |
||||
|
||||
--changing column storage should not impact the compression method |
||||
--but the data should not be compressed |
||||
ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar; |
||||
\d+ cmdata2 |
||||
Table "public.cmdata2" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | character varying | | | | extended | pglz | | |
||||
|
||||
ALTER TABLE cmdata2 ALTER COLUMN f1 SET STORAGE plain; |
||||
\d+ cmdata2 |
||||
Table "public.cmdata2" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+-------------------+-----------+----------+---------+---------+-------------+--------------+------------- |
||||
f1 | character varying | | | | plain | pglz | | |
||||
|
||||
INSERT INTO cmdata2 VALUES (repeat('123456789', 800)); |
||||
SELECT pg_column_compression(f1) FROM cmdata2; |
||||
pg_column_compression |
||||
----------------------- |
||||
|
||||
(1 row) |
||||
|
||||
-- test compression with materialized view |
||||
CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1; |
||||
\d+ mv |
||||
Materialized view "public.mv" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
x | text | | | | extended | pglz | | |
||||
View definition: |
||||
SELECT cmdata1.f1 AS x |
||||
FROM cmdata1; |
||||
|
||||
SELECT pg_column_compression(f1) FROM cmdata1; |
||||
pg_column_compression |
||||
----------------------- |
||||
lz4 |
||||
lz4 |
||||
(2 rows) |
||||
|
||||
SELECT pg_column_compression(x) FROM mv; |
||||
pg_column_compression |
||||
----------------------- |
||||
lz4 |
||||
lz4 |
||||
(2 rows) |
||||
|
||||
-- test compression with partition |
||||
CREATE TABLE cmpart(f1 text COMPRESSION lz4) PARTITION BY HASH(f1); |
||||
CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0); |
||||
CREATE TABLE cmpart2(f1 text COMPRESSION pglz); |
||||
ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1); |
||||
INSERT INTO cmpart VALUES (repeat('123456789', 1004)); |
||||
INSERT INTO cmpart VALUES (repeat('123456789', 4004)); |
||||
SELECT pg_column_compression(f1) FROM cmpart1; |
||||
pg_column_compression |
||||
----------------------- |
||||
lz4 |
||||
(1 row) |
||||
|
||||
SELECT pg_column_compression(f1) FROM cmpart2; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
(1 row) |
||||
|
||||
-- test compression with inheritence, error |
||||
CREATE TABLE cminh() INHERITS(cmdata, cmdata1); |
||||
NOTICE: merging multiple inherited definitions of column "f1" |
||||
ERROR: column "f1" has a compression method conflict |
||||
DETAIL: pglz versus lz4 |
||||
CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata); |
||||
NOTICE: merging column "f1" with inherited definition |
||||
ERROR: column "f1" has a compression method conflict |
||||
DETAIL: pglz versus lz4 |
||||
-- test default_toast_compression GUC |
||||
SET default_toast_compression = ''; |
||||
ERROR: invalid value for parameter "default_toast_compression": "" |
||||
DETAIL: default_toast_compression cannot be empty. |
||||
SET default_toast_compression = 'I do not exist compression'; |
||||
ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression" |
||||
DETAIL: Compression method "I do not exist compression" does not exist. |
||||
SET default_toast_compression = 'lz4'; |
||||
DROP TABLE cmdata2; |
||||
CREATE TABLE cmdata2 (f1 text); |
||||
\d+ cmdata2 |
||||
Table "public.cmdata2" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | text | | | | extended | lz4 | | |
||||
|
||||
-- test alter compression method |
||||
ALTER TABLE cmdata ALTER COLUMN f1 SET COMPRESSION lz4; |
||||
INSERT INTO cmdata VALUES (repeat('123456789', 4004)); |
||||
\d+ cmdata |
||||
Table "public.cmdata" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | text | | | | extended | lz4 | | |
||||
Indexes: |
||||
"idx" btree (f1) |
||||
|
||||
SELECT pg_column_compression(f1) FROM cmdata; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
lz4 |
||||
(2 rows) |
||||
|
||||
-- test alter compression method for the materialized view |
||||
ALTER MATERIALIZED VIEW mv ALTER COLUMN x SET COMPRESSION lz4; |
||||
\d+ mv |
||||
Materialized view "public.mv" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
x | text | | | | extended | lz4 | | |
||||
View definition: |
||||
SELECT cmdata1.f1 AS x |
||||
FROM cmdata1; |
||||
|
||||
-- test alter compression method for the partitioned table |
||||
ALTER TABLE cmpart1 ALTER COLUMN f1 SET COMPRESSION pglz; |
||||
ALTER TABLE cmpart2 ALTER COLUMN f1 SET COMPRESSION lz4; |
||||
-- new data should be compressed with the current compression method |
||||
INSERT INTO cmpart VALUES (repeat('123456789', 1004)); |
||||
INSERT INTO cmpart VALUES (repeat('123456789', 4004)); |
||||
SELECT pg_column_compression(f1) FROM cmpart1; |
||||
pg_column_compression |
||||
----------------------- |
||||
lz4 |
||||
pglz |
||||
(2 rows) |
||||
|
||||
SELECT pg_column_compression(f1) FROM cmpart2; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
lz4 |
||||
(2 rows) |
||||
|
||||
--vacuum full to recompress the data |
||||
SELECT pg_column_compression(f1) FROM cmdata; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
lz4 |
||||
(2 rows) |
||||
|
||||
VACUUM FULL cmdata; |
||||
SELECT pg_column_compression(f1) FROM cmdata; |
||||
pg_column_compression |
||||
----------------------- |
||||
lz4 |
||||
lz4 |
||||
(2 rows) |
||||
|
||||
-- check data is ok |
||||
SELECT length(f1) FROM cmdata; |
||||
length |
||||
-------- |
||||
10000 |
||||
36036 |
||||
(2 rows) |
||||
|
||||
SELECT length(f1) FROM cmdata1; |
||||
length |
||||
-------- |
||||
10040 |
||||
12449 |
||||
(2 rows) |
||||
|
||||
SELECT length(f1) FROM cmmove1; |
||||
length |
||||
-------- |
||||
10000 |
||||
(1 row) |
||||
|
||||
SELECT length(f1) FROM cmmove2; |
||||
length |
||||
-------- |
||||
10040 |
||||
(1 row) |
||||
|
||||
SELECT length(f1) FROM cmmove3; |
||||
length |
||||
-------- |
||||
10000 |
||||
10040 |
||||
(2 rows) |
||||
|
||||
\set HIDE_TOAST_COMPRESSION true |
||||
@ -0,0 +1,340 @@ |
||||
\set HIDE_TOAST_COMPRESSION false |
||||
-- test creating table with compression method |
||||
CREATE TABLE cmdata(f1 text COMPRESSION pglz); |
||||
CREATE INDEX idx ON cmdata(f1); |
||||
INSERT INTO cmdata VALUES(repeat('1234567890', 1000)); |
||||
\d+ cmdata |
||||
Table "public.cmdata" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | text | | | | extended | pglz | | |
||||
Indexes: |
||||
"idx" btree (f1) |
||||
|
||||
CREATE TABLE cmdata1(f1 TEXT COMPRESSION lz4); |
||||
ERROR: unsupported LZ4 compression method |
||||
DETAIL: This functionality requires the server to be built with lz4 support. |
||||
HINT: You need to rebuild PostgreSQL using --with-lz4. |
||||
INSERT INTO cmdata1 VALUES(repeat('1234567890', 1004)); |
||||
ERROR: relation "cmdata1" does not exist |
||||
LINE 1: INSERT INTO cmdata1 VALUES(repeat('1234567890', 1004)); |
||||
^ |
||||
\d+ cmdata1 |
||||
-- verify stored compression method in the data |
||||
SELECT pg_column_compression(f1) FROM cmdata; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
(1 row) |
||||
|
||||
SELECT pg_column_compression(f1) FROM cmdata1; |
||||
ERROR: relation "cmdata1" does not exist |
||||
LINE 1: SELECT pg_column_compression(f1) FROM cmdata1; |
||||
^ |
||||
-- decompress data slice |
||||
SELECT SUBSTR(f1, 200, 5) FROM cmdata; |
||||
substr |
||||
-------- |
||||
01234 |
||||
(1 row) |
||||
|
||||
SELECT SUBSTR(f1, 2000, 50) FROM cmdata1; |
||||
ERROR: relation "cmdata1" does not exist |
||||
LINE 1: SELECT SUBSTR(f1, 2000, 50) FROM cmdata1; |
||||
^ |
||||
-- copy with table creation |
||||
SELECT * INTO cmmove1 FROM cmdata; |
||||
\d+ cmmove1 |
||||
Table "public.cmmove1" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | text | | | | extended | pglz | | |
||||
|
||||
SELECT pg_column_compression(f1) FROM cmmove1; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
(1 row) |
||||
|
||||
-- copy to existing table |
||||
CREATE TABLE cmmove3(f1 text COMPRESSION pglz); |
||||
INSERT INTO cmmove3 SELECT * FROM cmdata; |
||||
INSERT INTO cmmove3 SELECT * FROM cmdata1; |
||||
ERROR: relation "cmdata1" does not exist |
||||
LINE 1: INSERT INTO cmmove3 SELECT * FROM cmdata1; |
||||
^ |
||||
SELECT pg_column_compression(f1) FROM cmmove3; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
(1 row) |
||||
|
||||
-- test LIKE INCLUDING COMPRESSION |
||||
CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION); |
||||
ERROR: relation "cmdata1" does not exist |
||||
LINE 1: CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION); |
||||
^ |
||||
\d+ cmdata2 |
||||
DROP TABLE cmdata2; |
||||
ERROR: table "cmdata2" does not exist |
||||
-- try setting compression for incompressible data type |
||||
CREATE TABLE cmdata2 (f1 int COMPRESSION pglz); |
||||
ERROR: column data type integer does not support compression |
||||
-- update using datum from different table |
||||
CREATE TABLE cmmove2(f1 text COMPRESSION pglz); |
||||
INSERT INTO cmmove2 VALUES (repeat('1234567890', 1004)); |
||||
SELECT pg_column_compression(f1) FROM cmmove2; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
(1 row) |
||||
|
||||
UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1; |
||||
ERROR: relation "cmdata1" does not exist |
||||
LINE 1: UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1; |
||||
^ |
||||
SELECT pg_column_compression(f1) FROM cmmove2; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
(1 row) |
||||
|
||||
-- test externally stored compressed data |
||||
CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS |
||||
'select array_agg(md5(g::text))::text from generate_series(1, 256) g'; |
||||
CREATE TABLE cmdata2 (f1 text COMPRESSION pglz); |
||||
INSERT INTO cmdata2 SELECT large_val() || repeat('a', 4000); |
||||
SELECT pg_column_compression(f1) FROM cmdata2; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
(1 row) |
||||
|
||||
INSERT INTO cmdata1 SELECT large_val() || repeat('a', 4000); |
||||
ERROR: relation "cmdata1" does not exist |
||||
LINE 1: INSERT INTO cmdata1 SELECT large_val() || repeat('a', 4000); |
||||
^ |
||||
SELECT pg_column_compression(f1) FROM cmdata1; |
||||
ERROR: relation "cmdata1" does not exist |
||||
LINE 1: SELECT pg_column_compression(f1) FROM cmdata1; |
||||
^ |
||||
SELECT SUBSTR(f1, 200, 5) FROM cmdata1; |
||||
ERROR: relation "cmdata1" does not exist |
||||
LINE 1: SELECT SUBSTR(f1, 200, 5) FROM cmdata1; |
||||
^ |
||||
SELECT SUBSTR(f1, 200, 5) FROM cmdata2; |
||||
substr |
||||
-------- |
||||
8f14e |
||||
(1 row) |
||||
|
||||
DROP TABLE cmdata2; |
||||
--test column type update varlena/non-varlena |
||||
CREATE TABLE cmdata2 (f1 int); |
||||
\d+ cmdata2 |
||||
Table "public.cmdata2" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- |
||||
f1 | integer | | | | plain | | | |
||||
|
||||
ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar; |
||||
\d+ cmdata2 |
||||
Table "public.cmdata2" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | character varying | | | | extended | pglz | | |
||||
|
||||
ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE int USING f1::integer; |
||||
\d+ cmdata2 |
||||
Table "public.cmdata2" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- |
||||
f1 | integer | | | | plain | | | |
||||
|
||||
--changing column storage should not impact the compression method |
||||
--but the data should not be compressed |
||||
ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar; |
||||
\d+ cmdata2 |
||||
Table "public.cmdata2" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | character varying | | | | extended | pglz | | |
||||
|
||||
ALTER TABLE cmdata2 ALTER COLUMN f1 SET STORAGE plain; |
||||
\d+ cmdata2 |
||||
Table "public.cmdata2" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+-------------------+-----------+----------+---------+---------+-------------+--------------+------------- |
||||
f1 | character varying | | | | plain | pglz | | |
||||
|
||||
INSERT INTO cmdata2 VALUES (repeat('123456789', 800)); |
||||
SELECT pg_column_compression(f1) FROM cmdata2; |
||||
pg_column_compression |
||||
----------------------- |
||||
|
||||
(1 row) |
||||
|
||||
-- test compression with materialized view |
||||
CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1; |
||||
ERROR: relation "cmdata1" does not exist |
||||
LINE 1: CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1; |
||||
^ |
||||
\d+ mv |
||||
SELECT pg_column_compression(f1) FROM cmdata1; |
||||
ERROR: relation "cmdata1" does not exist |
||||
LINE 1: SELECT pg_column_compression(f1) FROM cmdata1; |
||||
^ |
||||
SELECT pg_column_compression(x) FROM mv; |
||||
ERROR: relation "mv" does not exist |
||||
LINE 1: SELECT pg_column_compression(x) FROM mv; |
||||
^ |
||||
-- test compression with partition |
||||
CREATE TABLE cmpart(f1 text COMPRESSION lz4) PARTITION BY HASH(f1); |
||||
ERROR: unsupported LZ4 compression method |
||||
DETAIL: This functionality requires the server to be built with lz4 support. |
||||
HINT: You need to rebuild PostgreSQL using --with-lz4. |
||||
CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0); |
||||
ERROR: relation "cmpart" does not exist |
||||
CREATE TABLE cmpart2(f1 text COMPRESSION pglz); |
||||
ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1); |
||||
ERROR: relation "cmpart" does not exist |
||||
INSERT INTO cmpart VALUES (repeat('123456789', 1004)); |
||||
ERROR: relation "cmpart" does not exist |
||||
LINE 1: INSERT INTO cmpart VALUES (repeat('123456789', 1004)); |
||||
^ |
||||
INSERT INTO cmpart VALUES (repeat('123456789', 4004)); |
||||
ERROR: relation "cmpart" does not exist |
||||
LINE 1: INSERT INTO cmpart VALUES (repeat('123456789', 4004)); |
||||
^ |
||||
SELECT pg_column_compression(f1) FROM cmpart1; |
||||
ERROR: relation "cmpart1" does not exist |
||||
LINE 1: SELECT pg_column_compression(f1) FROM cmpart1; |
||||
^ |
||||
SELECT pg_column_compression(f1) FROM cmpart2; |
||||
pg_column_compression |
||||
----------------------- |
||||
(0 rows) |
||||
|
||||
-- test compression with inheritence, error |
||||
CREATE TABLE cminh() INHERITS(cmdata, cmdata1); |
||||
ERROR: relation "cmdata1" does not exist |
||||
CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata); |
||||
NOTICE: merging column "f1" with inherited definition |
||||
ERROR: column "f1" has a compression method conflict |
||||
DETAIL: pglz versus lz4 |
||||
-- test default_toast_compression GUC |
||||
SET default_toast_compression = ''; |
||||
ERROR: invalid value for parameter "default_toast_compression": "" |
||||
DETAIL: default_toast_compression cannot be empty. |
||||
SET default_toast_compression = 'I do not exist compression'; |
||||
ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression" |
||||
DETAIL: Compression method "I do not exist compression" does not exist. |
||||
SET default_toast_compression = 'lz4'; |
||||
ERROR: unsupported LZ4 compression method |
||||
DETAIL: This functionality requires the server to be built with lz4 support. |
||||
HINT: You need to rebuild PostgreSQL using --with-lz4. |
||||
DROP TABLE cmdata2; |
||||
CREATE TABLE cmdata2 (f1 text); |
||||
\d+ cmdata2 |
||||
Table "public.cmdata2" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | text | | | | extended | pglz | | |
||||
|
||||
-- test alter compression method |
||||
ALTER TABLE cmdata ALTER COLUMN f1 SET COMPRESSION lz4; |
||||
ERROR: unsupported LZ4 compression method |
||||
DETAIL: This functionality requires the server to be built with lz4 support. |
||||
HINT: You need to rebuild PostgreSQL using --with-lz4. |
||||
INSERT INTO cmdata VALUES (repeat('123456789', 4004)); |
||||
\d+ cmdata |
||||
Table "public.cmdata" |
||||
Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description |
||||
--------+------+-----------+----------+---------+----------+-------------+--------------+------------- |
||||
f1 | text | | | | extended | pglz | | |
||||
Indexes: |
||||
"idx" btree (f1) |
||||
|
||||
SELECT pg_column_compression(f1) FROM cmdata; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
pglz |
||||
(2 rows) |
||||
|
||||
-- test alter compression method for the materialized view |
||||
ALTER MATERIALIZED VIEW mv ALTER COLUMN x SET COMPRESSION lz4; |
||||
ERROR: relation "mv" does not exist |
||||
\d+ mv |
||||
-- test alter compression method for the partitioned table |
||||
ALTER TABLE cmpart1 ALTER COLUMN f1 SET COMPRESSION pglz; |
||||
ERROR: relation "cmpart1" does not exist |
||||
ALTER TABLE cmpart2 ALTER COLUMN f1 SET COMPRESSION lz4; |
||||
ERROR: unsupported LZ4 compression method |
||||
DETAIL: This functionality requires the server to be built with lz4 support. |
||||
HINT: You need to rebuild PostgreSQL using --with-lz4. |
||||
-- new data should be compressed with the current compression method |
||||
INSERT INTO cmpart VALUES (repeat('123456789', 1004)); |
||||
ERROR: relation "cmpart" does not exist |
||||
LINE 1: INSERT INTO cmpart VALUES (repeat('123456789', 1004)); |
||||
^ |
||||
INSERT INTO cmpart VALUES (repeat('123456789', 4004)); |
||||
ERROR: relation "cmpart" does not exist |
||||
LINE 1: INSERT INTO cmpart VALUES (repeat('123456789', 4004)); |
||||
^ |
||||
SELECT pg_column_compression(f1) FROM cmpart1; |
||||
ERROR: relation "cmpart1" does not exist |
||||
LINE 1: SELECT pg_column_compression(f1) FROM cmpart1; |
||||
^ |
||||
SELECT pg_column_compression(f1) FROM cmpart2; |
||||
pg_column_compression |
||||
----------------------- |
||||
(0 rows) |
||||
|
||||
--vacuum full to recompress the data |
||||
SELECT pg_column_compression(f1) FROM cmdata; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
pglz |
||||
(2 rows) |
||||
|
||||
VACUUM FULL cmdata; |
||||
SELECT pg_column_compression(f1) FROM cmdata; |
||||
pg_column_compression |
||||
----------------------- |
||||
pglz |
||||
pglz |
||||
(2 rows) |
||||
|
||||
-- check data is ok |
||||
SELECT length(f1) FROM cmdata; |
||||
length |
||||
-------- |
||||
10000 |
||||
36036 |
||||
(2 rows) |
||||
|
||||
SELECT length(f1) FROM cmdata1; |
||||
ERROR: relation "cmdata1" does not exist |
||||
LINE 1: SELECT length(f1) FROM cmdata1; |
||||
^ |
||||
SELECT length(f1) FROM cmmove1; |
||||
length |
||||
-------- |
||||
10000 |
||||
(1 row) |
||||
|
||||
SELECT length(f1) FROM cmmove2; |
||||
length |
||||
-------- |
||||
10040 |
||||
(1 row) |
||||
|
||||
SELECT length(f1) FROM cmmove3; |
||||
length |
||||
-------- |
||||
10000 |
||||
(1 row) |
||||
|
||||
\set HIDE_TOAST_COMPRESSION true |
||||
@ -0,0 +1,136 @@ |
||||
\set HIDE_TOAST_COMPRESSION false |
||||
|
||||
-- test creating table with compression method |
||||
CREATE TABLE cmdata(f1 text COMPRESSION pglz); |
||||
CREATE INDEX idx ON cmdata(f1); |
||||
INSERT INTO cmdata VALUES(repeat('1234567890', 1000)); |
||||
\d+ cmdata |
||||
CREATE TABLE cmdata1(f1 TEXT COMPRESSION lz4); |
||||
INSERT INTO cmdata1 VALUES(repeat('1234567890', 1004)); |
||||
\d+ cmdata1 |
||||
|
||||
-- verify stored compression method in the data |
||||
SELECT pg_column_compression(f1) FROM cmdata; |
||||
SELECT pg_column_compression(f1) FROM cmdata1; |
||||
|
||||
-- decompress data slice |
||||
SELECT SUBSTR(f1, 200, 5) FROM cmdata; |
||||
SELECT SUBSTR(f1, 2000, 50) FROM cmdata1; |
||||
|
||||
-- copy with table creation |
||||
SELECT * INTO cmmove1 FROM cmdata; |
||||
\d+ cmmove1 |
||||
SELECT pg_column_compression(f1) FROM cmmove1; |
||||
|
||||
-- copy to existing table |
||||
CREATE TABLE cmmove3(f1 text COMPRESSION pglz); |
||||
INSERT INTO cmmove3 SELECT * FROM cmdata; |
||||
INSERT INTO cmmove3 SELECT * FROM cmdata1; |
||||
SELECT pg_column_compression(f1) FROM cmmove3; |
||||
|
||||
-- test LIKE INCLUDING COMPRESSION |
||||
CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION); |
||||
\d+ cmdata2 |
||||
DROP TABLE cmdata2; |
||||
|
||||
-- try setting compression for incompressible data type |
||||
CREATE TABLE cmdata2 (f1 int COMPRESSION pglz); |
||||
|
||||
-- update using datum from different table |
||||
CREATE TABLE cmmove2(f1 text COMPRESSION pglz); |
||||
INSERT INTO cmmove2 VALUES (repeat('1234567890', 1004)); |
||||
SELECT pg_column_compression(f1) FROM cmmove2; |
||||
UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1; |
||||
SELECT pg_column_compression(f1) FROM cmmove2; |
||||
|
||||
-- test externally stored compressed data |
||||
CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS |
||||
'select array_agg(md5(g::text))::text from generate_series(1, 256) g'; |
||||
CREATE TABLE cmdata2 (f1 text COMPRESSION pglz); |
||||
INSERT INTO cmdata2 SELECT large_val() || repeat('a', 4000); |
||||
SELECT pg_column_compression(f1) FROM cmdata2; |
||||
INSERT INTO cmdata1 SELECT large_val() || repeat('a', 4000); |
||||
SELECT pg_column_compression(f1) FROM cmdata1; |
||||
SELECT SUBSTR(f1, 200, 5) FROM cmdata1; |
||||
SELECT SUBSTR(f1, 200, 5) FROM cmdata2; |
||||
DROP TABLE cmdata2; |
||||
|
||||
--test column type update varlena/non-varlena |
||||
CREATE TABLE cmdata2 (f1 int); |
||||
\d+ cmdata2 |
||||
ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar; |
||||
\d+ cmdata2 |
||||
ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE int USING f1::integer; |
||||
\d+ cmdata2 |
||||
|
||||
--changing column storage should not impact the compression method |
||||
--but the data should not be compressed |
||||
ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar; |
||||
\d+ cmdata2 |
||||
ALTER TABLE cmdata2 ALTER COLUMN f1 SET STORAGE plain; |
||||
\d+ cmdata2 |
||||
INSERT INTO cmdata2 VALUES (repeat('123456789', 800)); |
||||
SELECT pg_column_compression(f1) FROM cmdata2; |
||||
|
||||
-- test compression with materialized view |
||||
CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1; |
||||
\d+ mv |
||||
SELECT pg_column_compression(f1) FROM cmdata1; |
||||
SELECT pg_column_compression(x) FROM mv; |
||||
|
||||
-- test compression with partition |
||||
CREATE TABLE cmpart(f1 text COMPRESSION lz4) PARTITION BY HASH(f1); |
||||
CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0); |
||||
CREATE TABLE cmpart2(f1 text COMPRESSION pglz); |
||||
|
||||
ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1); |
||||
INSERT INTO cmpart VALUES (repeat('123456789', 1004)); |
||||
INSERT INTO cmpart VALUES (repeat('123456789', 4004)); |
||||
SELECT pg_column_compression(f1) FROM cmpart1; |
||||
SELECT pg_column_compression(f1) FROM cmpart2; |
||||
|
||||
-- test compression with inheritence, error |
||||
CREATE TABLE cminh() INHERITS(cmdata, cmdata1); |
||||
CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata); |
||||
|
||||
-- test default_toast_compression GUC |
||||
SET default_toast_compression = ''; |
||||
SET default_toast_compression = 'I do not exist compression'; |
||||
SET default_toast_compression = 'lz4'; |
||||
DROP TABLE cmdata2; |
||||
CREATE TABLE cmdata2 (f1 text); |
||||
\d+ cmdata2 |
||||
|
||||
-- test alter compression method |
||||
ALTER TABLE cmdata ALTER COLUMN f1 SET COMPRESSION lz4; |
||||
INSERT INTO cmdata VALUES (repeat('123456789', 4004)); |
||||
\d+ cmdata |
||||
SELECT pg_column_compression(f1) FROM cmdata; |
||||
|
||||
-- test alter compression method for the materialized view |
||||
ALTER MATERIALIZED VIEW mv ALTER COLUMN x SET COMPRESSION lz4; |
||||
\d+ mv |
||||
|
||||
-- test alter compression method for the partitioned table |
||||
ALTER TABLE cmpart1 ALTER COLUMN f1 SET COMPRESSION pglz; |
||||
ALTER TABLE cmpart2 ALTER COLUMN f1 SET COMPRESSION lz4; |
||||
|
||||
-- new data should be compressed with the current compression method |
||||
INSERT INTO cmpart VALUES (repeat('123456789', 1004)); |
||||
INSERT INTO cmpart VALUES (repeat('123456789', 4004)); |
||||
SELECT pg_column_compression(f1) FROM cmpart1; |
||||
SELECT pg_column_compression(f1) FROM cmpart2; |
||||
|
||||
--vacuum full to recompress the data |
||||
SELECT pg_column_compression(f1) FROM cmdata; |
||||
VACUUM FULL cmdata; |
||||
SELECT pg_column_compression(f1) FROM cmdata; |
||||
|
||||
-- check data is ok |
||||
SELECT length(f1) FROM cmdata; |
||||
SELECT length(f1) FROM cmdata1; |
||||
SELECT length(f1) FROM cmmove1; |
||||
SELECT length(f1) FROM cmmove2; |
||||
SELECT length(f1) FROM cmmove3; |
||||
|
||||
\set HIDE_TOAST_COMPRESSION true |
||||
Loading…
Reference in new issue