From d55240d24ce231fa61abf5945cb9122e81ca08ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20=C3=85strand?= Date: Tue, 19 Aug 2025 20:20:52 +0200 Subject: [PATCH 01/18] Bump pg_tde version to 2.0 This release is supposed to be 2.0. The SQL upgrade file is a dummy, but I believe it's required. --- contrib/pg_tde/Makefile | 2 +- contrib/pg_tde/expected/version.out | 2 +- contrib/pg_tde/meson.build | 1 + contrib/pg_tde/pg_tde--1.0--2.0.sql | 4 ++++ contrib/pg_tde/pg_tde.control | 2 +- contrib/pg_tde/src/include/pg_tde.h | 2 +- contrib/pg_tde/t/expected/basic.out | 2 +- 7 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 contrib/pg_tde/pg_tde--1.0--2.0.sql diff --git a/contrib/pg_tde/Makefile b/contrib/pg_tde/Makefile index 25726c46112..f691f3d4003 100644 --- a/contrib/pg_tde/Makefile +++ b/contrib/pg_tde/Makefile @@ -1,7 +1,7 @@ PGFILEDESC = "pg_tde access method" MODULE_big = pg_tde EXTENSION = pg_tde -DATA = pg_tde--1.0.sql +DATA = pg_tde--1.0--2.0.sql pg_tde--1.0.sql # Since meson supports skipping test suites this is a make only feature ifndef TDE_MODE diff --git a/contrib/pg_tde/expected/version.out b/contrib/pg_tde/expected/version.out index 44ebea8d04d..54e00d133bf 100644 --- a/contrib/pg_tde/expected/version.out +++ b/contrib/pg_tde/expected/version.out @@ -2,7 +2,7 @@ CREATE EXTENSION pg_tde; SELECT pg_tde_version(); pg_tde_version ---------------- - pg_tde 1.0.0 + pg_tde 2.0.0 (1 row) DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/meson.build b/contrib/pg_tde/meson.build index 0494fda2796..0899f2ebb61 100644 --- a/contrib/pg_tde/meson.build +++ b/contrib/pg_tde/meson.build @@ -78,6 +78,7 @@ endif install_data( 'pg_tde.control', 'pg_tde--1.0.sql', + 'pg_tde--1.0--2.0.sql', kwargs: contrib_data_args, ) diff --git a/contrib/pg_tde/pg_tde--1.0--2.0.sql b/contrib/pg_tde/pg_tde--1.0--2.0.sql new file mode 100644 index 00000000000..04d243f79af --- /dev/null +++ b/contrib/pg_tde/pg_tde--1.0--2.0.sql @@ -0,0 +1,4 @@ +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION pg_tde UPDATE TO '2.0'" to load this file. \quit + +-- This file is a dummy, because no SQL changed between 1.0 and 2.0 diff --git a/contrib/pg_tde/pg_tde.control b/contrib/pg_tde/pg_tde.control index 9ea82992d74..5496ed190ec 100644 --- a/contrib/pg_tde/pg_tde.control +++ b/contrib/pg_tde/pg_tde.control @@ -1,4 +1,4 @@ comment = 'pg_tde access method' -default_version = '1.0' +default_version = '2.0' module_pathname = '$libdir/pg_tde' relocatable = false diff --git a/contrib/pg_tde/src/include/pg_tde.h b/contrib/pg_tde/src/include/pg_tde.h index 4b6bb94d6d8..46e622d26a9 100644 --- a/contrib/pg_tde/src/include/pg_tde.h +++ b/contrib/pg_tde/src/include/pg_tde.h @@ -2,7 +2,7 @@ #define PG_TDE_H #define PG_TDE_NAME "pg_tde" -#define PG_TDE_VERSION "1.0.0" +#define PG_TDE_VERSION "2.0.0" #define PG_TDE_VERSION_STRING PG_TDE_NAME " " PG_TDE_VERSION #define PG_TDE_DATA_DIR "pg_tde" diff --git a/contrib/pg_tde/t/expected/basic.out b/contrib/pg_tde/t/expected/basic.out index 3947c0988f8..abeba5d06e2 100644 --- a/contrib/pg_tde/t/expected/basic.out +++ b/contrib/pg_tde/t/expected/basic.out @@ -20,7 +20,7 @@ CREATE EXTENSION pg_tde; SELECT extname, extversion FROM pg_extension WHERE extname = 'pg_tde'; extname | extversion ---------+------------ - pg_tde | 1.0 + pg_tde | 2.0 (1 row) CREATE TABLE test_enc (id SERIAL, k INTEGER, PRIMARY KEY (id)) USING tde_heap; From 50b3ec8e97b310dc63242190ca174ac0858b8e56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20=C3=85strand?= Date: Tue, 19 Aug 2025 20:26:11 +0200 Subject: [PATCH 02/18] Bump percona version to 17.5.3 This is the version we're about to release. --- ci_scripts/env.sh | 2 +- configure | 2 +- configure.ac | 2 +- meson.build | 2 +- src/test/modules/test_misc/t/008_percona_server_version.pl | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci_scripts/env.sh b/ci_scripts/env.sh index ec1cff4f8f2..11cf83eeed3 100644 --- a/ci_scripts/env.sh +++ b/ci_scripts/env.sh @@ -1,3 +1,3 @@ #!/bin/bash -export PERCONA_SERVER_VERSION=17.5.2 +export PERCONA_SERVER_VERSION=17.5.3 diff --git a/configure b/configure index 7960612b26b..542e7482173 100755 --- a/configure +++ b/configure @@ -2856,7 +2856,7 @@ cat >>confdefs.h <<_ACEOF _ACEOF -PG_PERCONAVERSION=2 +PG_PERCONAVERSION=3 cat >>confdefs.h <<_ACEOF #define PG_PERCONAVERSION "$PG_PERCONAVERSION" diff --git a/configure.ac b/configure.ac index 70fa3619d4b..d3f80558501 100644 --- a/configure.ac +++ b/configure.ac @@ -37,7 +37,7 @@ AC_DEFINE_UNQUOTED(PG_MAJORVERSION, "$PG_MAJORVERSION", [PostgreSQL major versio AC_DEFINE_UNQUOTED(PG_MAJORVERSION_NUM, $PG_MAJORVERSION, [PostgreSQL major version number]) AC_DEFINE_UNQUOTED(PG_MINORVERSION_NUM, $PG_MINORVERSION, [PostgreSQL minor version number]) -[PG_PERCONAVERSION=2] +[PG_PERCONAVERSION=3] AC_DEFINE_UNQUOTED(PG_PERCONAVERSION, "$PG_PERCONAVERSION", [PostgreSQL Percona version as a string]) PGAC_ARG_REQ(with, extra-version, [STRING], [append STRING to version], diff --git a/meson.build b/meson.build index c0542006604..8ea9d294a00 100644 --- a/meson.build +++ b/meson.build @@ -134,7 +134,7 @@ endif pg_version_major = pg_version_arr[0].to_int() pg_version_minor = pg_version_arr[1].to_int() pg_version_num = (pg_version_major * 10000) + pg_version_minor -pg_percona_ver = '2' +pg_percona_ver = '3' pg_url = 'https://www.postgresql.org/' diff --git a/src/test/modules/test_misc/t/008_percona_server_version.pl b/src/test/modules/test_misc/t/008_percona_server_version.pl index 347e173b523..8dc0ff0621d 100644 --- a/src/test/modules/test_misc/t/008_percona_server_version.pl +++ b/src/test/modules/test_misc/t/008_percona_server_version.pl @@ -17,7 +17,7 @@ my $pgdata = $node->data_dir; # To make this testcase work, PERCONA_SERVER_VERSION variable should be available in environment. # If you are using ci_scripts it is already declated in ci_scripts/env.sh # If you are using command line make for regression then export like: -# export PERCONA_SERVER_VERSION=17.5.2 +# export PERCONA_SERVER_VERSION=17.5.3 if (!defined($ENV{PERCONA_SERVER_VERSION})) { From 9d6297fa3053ebcfff616a35a574f90ad6390bd8 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Wed, 20 Aug 2025 10:58:39 +0300 Subject: [PATCH 03/18] Update the Features topic buttons for better clarity (#508) --- contrib/pg_tde/documentation/docs/features.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/pg_tde/documentation/docs/features.md b/contrib/pg_tde/documentation/docs/features.md index aa8703fc42b..3f51c25acea 100644 --- a/contrib/pg_tde/documentation/docs/features.md +++ b/contrib/pg_tde/documentation/docs/features.md @@ -19,4 +19,8 @@ The following features are available for the extension: * Table-level granularity for encryption and access control * Multiple [Key management options](global-key-provider-configuration/index.md) -[Learn more about TDE and pg_tde :material-arrow-right:](index/about-tde.md){.md-button} [Get started with installation :material-arrow-right:](install.md){.md-button} +## Next steps + +Learn more about how `pg_tde` implements Transparent Data Encryption: + +[About Transparent Data Encryption :material-arrow-right:](index/about-tde.md){.md-button} From c0ad12b50fca12db1700e37acfa5ff6904314648 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Fri, 22 Aug 2025 13:08:36 +0300 Subject: [PATCH 04/18] PG-1832 Document the archive and restore commands cont (#531) Continued from #523 - add pg_tde archive and restore commands - update cli-tools.md with paragraphs explaining New and extended tools - update pg-tde-restore-encrypt tool with new information and better descriptions for clarity - update the Features topic button for better clarity --- .../docs/command-line-tools/cli-tools.md | 19 +++++-- .../pg-tde-archive-decrypt.md | 55 +++++++++++++++++++ .../pg-tde-restore-encrypt.md | 49 +++++++++++++++++ contrib/pg_tde/documentation/mkdocs.yml | 6 +- 4 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-archive-decrypt.md create mode 100644 contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-restore-encrypt.md diff --git a/contrib/pg_tde/documentation/docs/command-line-tools/cli-tools.md b/contrib/pg_tde/documentation/docs/command-line-tools/cli-tools.md index e0edeb4b5de..aac0c8d2f06 100644 --- a/contrib/pg_tde/documentation/docs/command-line-tools/cli-tools.md +++ b/contrib/pg_tde/documentation/docs/command-line-tools/cli-tools.md @@ -1,7 +1,18 @@ # Overview of pg_tde CLI tools -The `pg_tde` extension introduces new command-line utilities and extends some existing PostgreSQL tools to support encrypted WAL and tables. These include: +The `pg_tde` extension introduces new command-line utilities and extends some existing PostgreSQL tools to support encrypted WAL and tables. -* [pg_tde_change_key_provider](../command-line-tools/pg-tde-change-key-provider.md): change encryption key provider for a database -* [pg_waldump](../command-line-tools/pg-waldump.md): inspect and decrypt WAL files -* [pg_checksums](../command-line-tools/pg-tde-checksums.md): verify data checksums (non-encrypted files only) +## New tools + +These tools are introduced by `pg_tde` to support key rotation and WAL encryption workflows: + +* [pg_tde_change_key_provider](./pg-tde-change-key-provider.md): change the encryption key provider for a database +* [pg_tde_archive_decrypt](./pg-tde-archive-decrypt.md): decrypts WAL before archiving +* [pg_tde_restore_encrypt](./pg-tde-restore-encrypt.md): a custom restore command for making sure the restored WAL is encrypted + +## Extended tools + +These existing PostgreSQL tools are enhanced to support `pg_tde`: + +* [pg_checksums](./pg-tde-checksums.md): verify data checksums (non-encrypted files only) +* [pg_waldump](./pg-waldump.md): inspect and decrypt WAL files diff --git a/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-archive-decrypt.md b/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-archive-decrypt.md new file mode 100644 index 00000000000..0b63ab5212d --- /dev/null +++ b/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-archive-decrypt.md @@ -0,0 +1,55 @@ +# pg_tde_archive_decrypt + +The `pg_tde_archive_decrypt` tool wraps an archive command and decrypts WAL files before archiving. It allows external tools to access unencrypted WAL data, which is required because WAL encryption keys in the two-key hierarchy are host-specific and may not be available on the replay host. + +!!! tip + + For more information on the encryption architecture and key hierarchy, see [Architecture](../architecture/architcture.md). + +This tool is often used in conjunction with [pg_tde_restore_encrypt](./pg-tde-restore-encrypt.md) to support WAL archive. + +## How it works + +1. Decrypts the WAL segment to a temporary file on a RAM disk (`/dev/shm`) +2. Replaces `%p` and `%f` in the archive command with the path and name of the decrypted file +3. Executes the archive command + +!!! note + + To ensure security, encrypt the files stored in your WAL archive using tools like `PgBackRest`. + +## Usage + +```bash +pg_tde_archive_decrypt [OPTION] +pg_tde_archive_decrypt DEST-NAME SOURCE-PATH ARCHIVE-COMMAND +``` + +## Parameter descriptions + +* `DEST-NAME`: name of the WAL file to send to the archive +* `SOURCE-PATH`: path to the original encrypted WAL file +* `ARCHIVE-COMMAND`: archive command to wrap. `%p` and `%f` are replaced with the decrypted WAL file path and WAL file name, respectively. + +## Options + +* `-V, --version`: show version information, then exit +* `-?, --help`: show help information, then exit + +!!! note + + Any `%f` or `%p` parameter in `ARCHIVE-COMMAND` has to be escaped as `%%f` or `%%p` respectively if used as `archive_command` in `postgresql.conf`. + +## Examples + +### Using `cp` + +```ini +archive_command='pg_tde_archive_decrypt %f %p "cp %%p /mnt/server/archivedir/%%f"' +``` + +### Using `PgBackRest` + +```ini +archive_command='pg_tde_archive_decrypt %f %p "pgbackrest --stanza=your_stanza archive-push %%p"' +``` diff --git a/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-restore-encrypt.md b/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-restore-encrypt.md new file mode 100644 index 00000000000..7435de2d0b7 --- /dev/null +++ b/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-restore-encrypt.md @@ -0,0 +1,49 @@ +# pg_tde_restore_encrypt + +The `pg_tde_restore_encrypt` tool wraps a normal restore command from the WAL archive and writes them to disk in a format compatible with `pg_tde`. + +!!! note + + This command is often use together with [pg_tde_archive_decrypt](./pg-tde-archive-decrypt.md). + +## How it works + +1. Replaces `%f` and `%p` in the restore command with the WAL file name and temporary file path (in `/dev/shm`) +2. Runs the restore command to fetch the unencrypted WAL from the archive and write it to the temp file +3. Encrypts the temp file and writes it to the destination path in PostgreSQL’s data directory + +## Usage + +```bash +pg_tde_restore_encrypt [OPTION] +pg_tde_restore_encrypt SOURCE-NAME DEST-PATH RESTORE-COMMAND +``` + +## Parameter descriptions + +* `SOURCE-NAME`: name of the WAL file to retrieve from the archive +* `DEST-PATH`: path where the encrypted WAL file should be written +* `RESTORE-COMMAND`: restore command to wrap; `%p` and `%f` are replaced with the WAL file name and path to write the unencrypted WAL, respectively + +## Options + +* `-V, --version`: show version information, then exit +* `-?, --help`: show help information, then exit + +!!! note + + Any `%f` or `%p` parameter in `RESTORE-COMMAND` has to be escaped as `%%f` or `%%p` respectively if used as `restore_command` in `postgresql.conf`. + +## Examples + +### Using `cp` + +```ini +restore_command='pg_tde_restore_encrypt %f %p "cp /mnt/server/archivedir/%%f %%p"' +``` + +### Using `PgBackRest` + +```ini +restore_command='pg_tde_restore_encrypt %f %p "pgbackrest --stanza=your_stanza archive-get %%f \"%%p\""' +``` diff --git a/contrib/pg_tde/documentation/mkdocs.yml b/contrib/pg_tde/documentation/mkdocs.yml index 43725bb5de5..39c1c4eec03 100644 --- a/contrib/pg_tde/documentation/mkdocs.yml +++ b/contrib/pg_tde/documentation/mkdocs.yml @@ -193,11 +193,13 @@ nav: - "Functions": functions.md - "Streaming Replication with tde_heap": replication.md - "TDE operations": - - "pg_tde CLI Tools": + - "pg_tde CLI tools": - "Overview": command-line-tools/cli-tools.md - "pg_tde_change_key_provider": command-line-tools/pg-tde-change-key-provider.md - - "pg_waldump": command-line-tools/pg-waldump.md + - "pg_tde_archive_decrypt": command-line-tools/pg-tde-archive-decrypt.md + - "pg_tde_restore_encrypt": command-line-tools/pg-tde-restore-encrypt.md - "pg_checksums": command-line-tools/pg-tde-checksums.md + - "pg_waldump": command-line-tools/pg-waldump.md - "Uninstall pg_tde": how-to/uninstall.md - "Configure Multi-tenancy": how-to/multi-tenant-setup.md - "Encryption Enforcement": how-to/enforcement.md From 307b33d65625e98afd5f19841691071522a6cde5 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Fri, 22 Aug 2025 13:11:22 +0300 Subject: [PATCH 05/18] Add WAL content for 2.0 release (#499) - remove (tech preview) - remove mentions of WAL being BETA and warning notes - add WAL tool support to limitations, improve flow, add button to setup - add limitation regarding WAL shipping standy not supported with WAL encryption - add mention of open source and enterprise ed being supported for pg_tde - add none method to basebackup and link to topic - add Example Patroni configuration for Patroni tool - improve supported vs unsupported tools section in Limitations --- .../docs/architecture/architecture.md | 5 +- .../docs/command-line-tools/pg-waldump.md | 3 - contrib/pg_tde/documentation/docs/faq.md | 5 +- .../pg_tde/documentation/docs/functions.md | 5 +- .../overview.md | 15 ++- .../set-principal-key.md | 12 ++- .../vault.md | 2 +- .../docs/index/how-does-tde-work.md | 10 +- .../docs/index/tde-limitations.md | 97 ++++++++++++++++++- contrib/pg_tde/documentation/docs/test.md | 2 +- .../documentation/docs/wal-encryption.md | 7 +- contrib/pg_tde/documentation/mkdocs.yml | 2 +- 12 files changed, 126 insertions(+), 39 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/architecture/architecture.md b/contrib/pg_tde/documentation/docs/architecture/architecture.md index c06dcb55161..aee77772ece 100644 --- a/contrib/pg_tde/documentation/docs/architecture/architecture.md +++ b/contrib/pg_tde/documentation/docs/architecture/architecture.md @@ -20,7 +20,7 @@ The following sections break down the key architectural components of this desig * Indexes * Sequences * Temporary tables -* Write Ahead Log (WAL), still in beta. **Do not enable this feature in production environments**. +* Write Ahead Log (WAL) **Extension** means that `pg_tde` should be implemented only as an extension, possibly compatible with any PostgreSQL distribution, including the open source community version. This requires changes in the PostgreSQL core to make it more extensible. Therefore, `pg_tde` currently works only with the [Percona Server for PostgreSQL](https://docs.percona.com/postgresql/17/index.html) - a binary replacement of community PostgreSQL and included in Percona Distribution for PostgreSQL. @@ -82,9 +82,6 @@ Later decisions are made using a slightly modified Storage Manager (SMGR) API: w ### WAL encryption -!!! note - The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. - WAL encryption is controlled globally via a global GUC variable, `pg_tde.wal_encrypt`, that requires a server restart. WAL keys also contain the [LSN](https://www.postgresql.org/docs/17/wal-internals.html) of the first WAL write after key creation. This allows `pg_tde` to know which WAL ranges are encrypted or not and with which key. diff --git a/contrib/pg_tde/documentation/docs/command-line-tools/pg-waldump.md b/contrib/pg_tde/documentation/docs/command-line-tools/pg-waldump.md index 0362eadcef7..a3ac50df4df 100644 --- a/contrib/pg_tde/documentation/docs/command-line-tools/pg-waldump.md +++ b/contrib/pg_tde/documentation/docs/command-line-tools/pg-waldump.md @@ -2,9 +2,6 @@ [`pg_waldump` :octicons-link-external-16:](https://www.postgresql.org/docs/current/pgwaldump.html) is a tool to display a human-readable rendering of the Write-Ahead Log (WAL) of a PostgreSQL database cluster. -!!! warning - The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. - To read encrypted WAL records, `pg_waldump` supports the following additional arguments: * `keyring_path` is the directory where the keyring configuration files for WAL are stored. The following files are included: diff --git a/contrib/pg_tde/documentation/docs/faq.md b/contrib/pg_tde/documentation/docs/faq.md index b2448183711..9e9c88f181a 100644 --- a/contrib/pg_tde/documentation/docs/faq.md +++ b/contrib/pg_tde/documentation/docs/faq.md @@ -94,9 +94,6 @@ The principal key is used to encrypt the internal keys. The principal key is sto ### WAL encryption -!!! note - WAL encryption is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. - WAL encryption is done globally for the entire database cluster. All modifications to any database within a PostgreSQL cluster are written to the same WAL to maintain data consistency and integrity and ensure that PostgreSQL cluster can be restored to a consistent state. Therefore, WAL is encrypted globally. When you turn on WAL encryption, `pg_tde` encrypts entire WAL files starting from the first WAL write after the server was started with the encryption turned on. @@ -140,7 +137,7 @@ Since the `SET ACCESS METHOD` command drops hint bits and this may affect the pe You must restart the database in the following cases to apply the changes: * after you enabled the `pg_tde` extension -* when enabling WAL encryption, which is currently in beta. **Do not enable this feature in production environments**. +* when enabling WAL encryption After that, no database restart is required. When you create or alter the table using the `tde_heap` access method, the files are marked as those that require encryption. The encryption happens at the storage manager level, before a transaction is written to disk. Read more about [how tde_heap works](index/table-access-method.md#how-tde_heap-works-with-pg_tde). diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index b290d440ee7..82e6e3449a2 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -242,7 +242,7 @@ SELECT pg_tde_create_key_using_database_key_provider( ### pg_tde_create_key_using_global_key_provider -Creates a principal key at a global key provider with the given name. Use this key later with the `pg_tde_set_` series of functions. +Creates a principal key at a global key provider with the given name. Use this key later with the `pg_tde_set_*` series of functions. ```sql SELECT pg_tde_create_key_using_global_key_provider( @@ -286,9 +286,6 @@ SELECT pg_tde_set_server_key_using_global_key_provider( ); ``` -!!! warning - The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. - The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: * If set to `true`, a new key must be unique. diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/overview.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/overview.md index 800f5059257..14aa13e569c 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/overview.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/overview.md @@ -9,10 +9,15 @@ To use an external KMS with `pg_tde`, follow these two steps: 2. Set the [Global Principal Key](set-principal-key.md) !!! note - While keyfiles may be acceptable for **local** or **testing environments**, KMS integration is the recommended approach for production deployments. + While key files may be acceptable for **local** or **testing environments**, KMS integration is the recommended approach for production deployments. -Select your prefered configuration from the links below: +`pg_tde` has been tested with the following key providers: -[KMIP Configuration :material-arrow-right:](kmip-server.md){.md-button} -[Vault Configuration :material-arrow-right:](vault.md){.md-button} -[Keyring File Configuration (not recommended) :material-arrow-right:](keyring.md){.md-button} +| KMS Provider | Description | Documentation | +|--------------------|-------------------------------------------------------|---------------| +| **KMIP** | Standard Key Management Interoperability Protocol. | [Configure KMIP →](kmip-server.md) | +| **Vault** | HashiCorp Vault integration (KV v2 API, KMIP engine). | [Configure Vault →](vault.md) | +| **Fortanix** | Fortanix DSM key management. | [Configure Fortanix →](kmip-fortanix.md) | +| **Thales** | Thales CipherTrust Manager and DSM. | [Configure Thales →](kmip-thales.md) | +| **OpenBao** | Community fork of Vault, supporting KV v2. | [Configure OpenBao →](kmip-openbao.md) | +| **Keyring file** *(not recommended)* | Local key file for dev/test only. | [Configure keyring file →](keyring.md) | diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md index 1d4e55788d5..781ce5869dd 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md @@ -1,10 +1,16 @@ # Global Principal Key configuration -You can configure a default principal key using a global key provider. This key will be used by all databases that do not have their own encryption keys configured. The function **both** sets the principal key and rotates internal keys as needed. +You can configure a default principal key using a global key provider. This key is used by all databases that do not have their own encryption keys configured. + +There are two main functions for this: + +- [pg_tde_create_key_using_global_key_provider()](../functions.md#pg_tde_create_key_using_global_key_provider) creates a principal key at a global key provider +- [pg_tde_set_default_key_using_global_key_provider()](../functions.md#pg_tde_set_default_key_using_global_key_provider) sets the default principal key and rotates the internal encryption key if one is already configured ## Create a default principal key !!! note + The sample output below is for demonstration purposes only. Be sure to replace the key name and provider with your actual values. To create a global principal key, run: @@ -57,6 +63,7 @@ SELECT pg_tde_set_default_key_using_global_key_provider( * `global_vault_provider` is the name of the global key provider you previously configured. !!! note + If no error is reported, the action completed successfully. ## How key generation works @@ -64,8 +71,11 @@ SELECT pg_tde_set_default_key_using_global_key_provider( The key material (actual cryptographic key) is auto-generated by `pg_tde` and stored securely by the configured provider. !!! note + This process sets the **default principal key for the entire server**. Any database without a key explicitly configured will fall back to this key. ## Next steps +To confirm that encryption is working as expected, follow the validation steps: + [Validate Encryption with pg_tde :material-arrow-right:](../test.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md index 00bac8074d4..ee913bc991d 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md @@ -1,6 +1,6 @@ # Vault configuration -You can configure `pg_tde` to use HashiCorp Vault as a global key provider for managing encryption keys securely. +You can configure `pg_tde` to use HashiCorp Vault as a global key provider for managing encryption keys securely. Both the open source and enterprise editions are supported. !!! note This guide assumes that your Vault server is already set up and accessible. Vault configuration is outside the scope of this document, see [Vault's official documentation](https://developer.hashicorp.com/vault/docs) for more information. diff --git a/contrib/pg_tde/documentation/docs/index/how-does-tde-work.md b/contrib/pg_tde/documentation/docs/index/how-does-tde-work.md index 8bce6c474c0..64e422e4856 100644 --- a/contrib/pg_tde/documentation/docs/index/how-does-tde-work.md +++ b/contrib/pg_tde/documentation/docs/index/how-does-tde-work.md @@ -6,15 +6,9 @@ To encrypt the data, two types of keys are used: * The **principal key** to encrypt database keys. It is kept separately from the database keys and is managed externally in the key management store. !!! note + For more information on managing and storing principal keys externally, including supported key management systems and the local keyring option, see [Key management overview](../global-key-provider-configuration/overview.md). - For more information on managing and storing principal keys externally, see [Configure Global Key Provider](../global-key-provider-configuration/overview.md). - -You have the following options to store and manage principal keys externally: - -* Use the HashiCorp Vault server. Only the back end KV Secrets Engine - Version 2 (API) is supported. -* Use the KMIP-compatible server. `pg_tde` has been tested with the [PyKMIP](https://pykmip.readthedocs.io/en/latest/server.html) server and [the HashiCorp Vault Enterprise KMIP Secrets Engine](https://www.vaultproject.io/docs/secrets/kmip). - -The encryption process is the following: +The encryption process works as follows: ![image](../_images/tde-flow.png) diff --git a/contrib/pg_tde/documentation/docs/index/tde-limitations.md b/contrib/pg_tde/documentation/docs/index/tde-limitations.md index 8e3376671ae..5b4e4328722 100644 --- a/contrib/pg_tde/documentation/docs/index/tde-limitations.md +++ b/contrib/pg_tde/documentation/docs/index/tde-limitations.md @@ -1,8 +1,99 @@ # Limitations of pg_tde -The following are current limitations of `pg_tde`: +Limitations of `pg_tde` {{release}}: -* System tables, which include statistics data and database statistics, are currently **not encrypted**. -* The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. +* PostgreSQL’s internal system tables, which include statistics and metadata, are not encrypted. +* Temporary files created when queries exceed `work_mem` are not encrypted. These files may persist during long-running queries or after a server crash which can expose sensitive data in plaintext on disk. + +## Currently unsupported WAL tools + +The following tools are currently unsupported with `pg_tde` WAL encryption: + +* `pg_createsubscriber` +* `pg_verifybackup` (checksum mismatch with encrypted WAL) + +The following tools and extensions in Percona Distribution for PostgreSQL have been tested and verified to work with `pg_tde` WAL encryption: + +## Supported WAL tools + +The following tools have been tested and verified by Percona to work with `pg_tde` WAL encryption: + +* Patroni, for an example configuration see the following [Patroni configuration file](#example-patroni-configuration) +* `pg_basebackup` (with `--wal-method=stream` or `--wal-method=none`), for details on using `pg_basebackup` with WAL encryption, see [Backup with WAL encryption enabled](../how-to/backup-wal-enabled.md) +* `pg_resetwal` +* `pg_rewind` +* `pg_upgrade` +* `pg_waldump` +* pgBackRest + +## Example Patroni configuration + +The following is a Percona-tested example configuration. + +??? example "Click to expand the Percona-tested Patroni configuration" + ```yaml + # Example Patroni configuration file maintained by Percona + # Source: https://github.com/jobinau/pgscripts/blob/main/patroni/patroni.yml + scope: postgres + namespace: /db/ + name: postgresql0 + + restapi: + listen: 0.0.0.0:8008 + connect_address: 127.0.0.1:8008 + + etcd: + host: 127.0.0.1:2379 + + bootstrap: + dcs: + ttl: 30 + loop_wait: 10 + retry_timeout: 10 + maximum_lag_on_failover: 1048576 + postgresql: + use_pg_rewind: true + use_slots: true + parameters: + max_connections: 100 + shared_buffers: 1GB + wal_level: replica + hot_standby: "on" + wal_keep_size: 256MB + max_wal_senders: 10 + max_replication_slots: 10 + + initdb: + - encoding: UTF8 + - data-checksums + + pg_hba: + - host replication replicator 127.0.0.1/32 md5 + - host all all 0.0.0.0/0 md5 + + postgresql: + listen: 0.0.0.0:5432 + connect_address: 127.0.0.1:5432 + data_dir: /var/lib/postgresql/data + bin_dir: /usr/lib/postgresql/14/bin + authentication: + replication: + username: replicator + password: rep-pass + superuser: + username: postgres + password: secretpassword + ``` + +!!! warning + The above example is Percona-tested, but Patroni versions differ, especially with discovery backends such as `etcd`. Ensure you adjust the configuration to match your environment, version, and security requirements. + +## Next steps + +Check which PostgreSQL versions and deployment types are compatible with `pg_tde` before planning your installation. [View the versions and supported deployments :material-arrow-right:](supported-versions.md){.md-button} + +Begin the installation process when you're ready to set up encryption. + +[Start installing `pg_tde`](../install.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/test.md b/contrib/pg_tde/documentation/docs/test.md index be73e73863e..e4578e6bf6e 100644 --- a/contrib/pg_tde/documentation/docs/test.md +++ b/contrib/pg_tde/documentation/docs/test.md @@ -59,4 +59,4 @@ ALTER TABLE table_name SET ACCESS METHOD tde_heap; ## Next steps -[Configure WAL Encryption (tech preview) :material-arrow-right:](wal-encryption.md){.md-button} +[Configure WAL encryption :material-arrow-right:](wal-encryption.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/wal-encryption.md b/contrib/pg_tde/documentation/docs/wal-encryption.md index 33547d0e359..6de83230c11 100644 --- a/contrib/pg_tde/documentation/docs/wal-encryption.md +++ b/contrib/pg_tde/documentation/docs/wal-encryption.md @@ -1,7 +1,4 @@ -# Configure WAL Encryption (tech preview) - -!!! warning - The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. +# Configure WAL encryption Before enabling WAL encryption, follow the steps below to create a principal key and configure it for WAL: @@ -118,3 +115,5 @@ Now WAL files start to be encrypted for both encrypted and unencrypted tables. For more technical references related to architecture, variables or functions, see: [Technical Reference](advanced-topics/tech-reference.md){.md-button} + +💬 Need help customizing this for your infrastructure? [Contact Percona support :octicons-link-external-16:](get-help.md) \ No newline at end of file diff --git a/contrib/pg_tde/documentation/mkdocs.yml b/contrib/pg_tde/documentation/mkdocs.yml index 39c1c4eec03..388d3672fd9 100644 --- a/contrib/pg_tde/documentation/mkdocs.yml +++ b/contrib/pg_tde/documentation/mkdocs.yml @@ -185,7 +185,7 @@ nav: - "Keyring file configuration": global-key-provider-configuration/keyring.md - "2.2 Global Principal Key configuration": global-key-provider-configuration/set-principal-key.md - "3. Validate encryption with pg_tde": test.md - - "4. Configure WAL encryption (tech preview)": wal-encryption.md + - "4. Configure WAL encryption": wal-encryption.md - "Technical reference": - "Overview": advanced-topics/tech-reference.md - "Architecture": architecture/architecture.md From acaddab9ab83da479589ecef49892c5823042770 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Fri, 22 Aug 2025 13:40:27 +0300 Subject: [PATCH 06/18] PG-1858 Document backing up with WAL with encrypt enabled (#534) - add new topic called # Backing up with WAL encryption enabled - add two suptopics for other wal methods and restore backup created with wal encrypt - reword to short form option flags --- .../docs/how-to/backup-wal-enabled.md | 44 +++++++++++++++++++ contrib/pg_tde/documentation/mkdocs.yml | 1 + 2 files changed, 45 insertions(+) create mode 100644 contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md diff --git a/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md b/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md new file mode 100644 index 00000000000..25a5342b1b4 --- /dev/null +++ b/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md @@ -0,0 +1,44 @@ +# Backup with WAL encryption enabled + +To create a backup with WAL encryption enabled: + +1. Copy the `pg_tde` directory from the source server’s data directory, for example `/var/lib/postgresql/data/pg_tde/`, including the `wal_keys` and `1664_providers` files, to the backup destination directory where `pg_basebackup` will write the backup. + +Also copy any external files referenced by your providers configuration (such as certificate or key files) into the same relative paths under the backup destination, so that they are located and validated by `pg_basebackup -E`. + +2. Run: + + ```bash + pg_basebackup -D /path/to/backup -E + ``` + + Where: + + - `-D /path/to/backup` specifies the backup location where you have to copy `pg_tde` + - `-E` (or `--encrypt-wal`) enables WAL encryption and validates that the copied `pg_tde` and provider files are present and that the server key is accessible (required) + +!!! note + - The `-E` flag only works with the `-X stream` option (default). It is not compatible with `-X none` or `-X fetch`. For more information, see [the other WAL methods topic](#other-wal-methods). + - The `-E` flag is only supported with the plain output format (`-F p`). It cannot be used with the tar output format (`-F t`). + +## Restore a backup created with WAL encryption + +When you want to restore a backup created with `pg_basebackup -E`: + +1. Ensure all external files referenced by your providers configuration (such as certificates or key files) are also present and accessible at the same relative paths. +2. Start PostgreSQL with the restored data directory. + +## Other WAL methods + +The `-X fetch` option works with encrypted WAL without requiring any additional flags. +The `-X none` option excludes WAL from the backup and is unaffected by WAL encryption. + +If the source server has `pg_tde/wal_keys`, running `pg_basebackup` with `-X none` or `-X fetch` produces warnings such as: + +```sql +pg_basebackup: warning: the source has WAL keys, but no WAL encryption configured for the target backups +pg_basebackup: detail: This may lead to exposed data and broken backup +pg_basebackup: hint: Run pg_basebackup with -E to encrypt streamed WAL +``` + +You can safely ignore these warnings when using `-X none` or `-X fetch`, since in both cases WAL is not streamed. diff --git a/contrib/pg_tde/documentation/mkdocs.yml b/contrib/pg_tde/documentation/mkdocs.yml index 388d3672fd9..5ffd25e3236 100644 --- a/contrib/pg_tde/documentation/mkdocs.yml +++ b/contrib/pg_tde/documentation/mkdocs.yml @@ -204,6 +204,7 @@ nav: - "Configure Multi-tenancy": how-to/multi-tenant-setup.md - "Encryption Enforcement": how-to/enforcement.md - "Decrypt an Encrypted Table": how-to/decrypt.md + - "Backup with WAL encryption enabled": how-to/backup-wal-enabled.md - "Restore an encrypted pg_tde backup": how-to/restore-backups.md - faq.md - "Release notes": From 6719db5704a43e2274cb9d0e965142e5391b42fe Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Mon, 25 Aug 2025 19:55:25 +0300 Subject: [PATCH 07/18] Update WAL backup topic regarding TAR not supporting `-X stream` when WAL encryption is enabled (#548) - renamed title, reorg content for easier scanning - turned into note the `pg_tde/wal_keys` at the end --- .../pg-tde-archive-decrypt.md | 6 ++- .../pg-tde-restore-encrypt.md | 3 ++ .../docs/how-to/backup-wal-enabled.md | 41 +++++++++++++------ 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-archive-decrypt.md b/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-archive-decrypt.md index 0b63ab5212d..26a80eda805 100644 --- a/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-archive-decrypt.md +++ b/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-archive-decrypt.md @@ -3,8 +3,7 @@ The `pg_tde_archive_decrypt` tool wraps an archive command and decrypts WAL files before archiving. It allows external tools to access unencrypted WAL data, which is required because WAL encryption keys in the two-key hierarchy are host-specific and may not be available on the replay host. !!! tip - - For more information on the encryption architecture and key hierarchy, see [Architecture](../architecture/architcture.md). + For more information on the encryption architecture and key hierarchy, see [Architecture](../architecture/architecture.md). This tool is often used in conjunction with [pg_tde_restore_encrypt](./pg-tde-restore-encrypt.md) to support WAL archive. @@ -53,3 +52,6 @@ archive_command='pg_tde_archive_decrypt %f %p "cp %%p /mnt/server/archivedir/%%f ```ini archive_command='pg_tde_archive_decrypt %f %p "pgbackrest --stanza=your_stanza archive-push %%p"' ``` + +!!! warning + When using PgBackRest with WAL encryption, disable PostgreSQL data checksums. Otherwise, PgBackRest may spam error messages, and in some package builds the log statement can cause crashes. diff --git a/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-restore-encrypt.md b/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-restore-encrypt.md index 7435de2d0b7..8ea0993404c 100644 --- a/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-restore-encrypt.md +++ b/contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-restore-encrypt.md @@ -47,3 +47,6 @@ restore_command='pg_tde_restore_encrypt %f %p "cp /mnt/server/archivedir/%%f %%p ```ini restore_command='pg_tde_restore_encrypt %f %p "pgbackrest --stanza=your_stanza archive-get %%f \"%%p\""' ``` + +!!! warning + When using PgBackRest with WAL encryption, disable PostgreSQL data checksums. Otherwise, PgBackRest may spam error messages, and in some package builds the log statement can cause crashes. diff --git a/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md b/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md index 25a5342b1b4..a3214d9a6a3 100644 --- a/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md +++ b/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md @@ -14,8 +14,8 @@ Also copy any external files referenced by your providers configuration (such as Where: - - `-D /path/to/backup` specifies the backup location where you have to copy `pg_tde` - - `-E` (or `--encrypt-wal`) enables WAL encryption and validates that the copied `pg_tde` and provider files are present and that the server key is accessible (required) + - `-D /path/to/backup` specifies the backup location where you have to copy `pg_tde`. + - `-E` (or `--encrypt-wal`) enables WAL encryption and validates that the copied `pg_tde` and provider files are present and that the server key is accessible (required). !!! note - The `-E` flag only works with the `-X stream` option (default). It is not compatible with `-X none` or `-X fetch`. For more information, see [the other WAL methods topic](#other-wal-methods). @@ -28,17 +28,34 @@ When you want to restore a backup created with `pg_basebackup -E`: 1. Ensure all external files referenced by your providers configuration (such as certificates or key files) are also present and accessible at the same relative paths. 2. Start PostgreSQL with the restored data directory. -## Other WAL methods +## Backup method compatibility with WAL encryption -The `-X fetch` option works with encrypted WAL without requiring any additional flags. -The `-X none` option excludes WAL from the backup and is unaffected by WAL encryption. +Tar format (`-F t`): -If the source server has `pg_tde/wal_keys`, running `pg_basebackup` with `-X none` or `-X fetch` produces warnings such as: +* Works with `-X fetch`. +* Does not support `-X stream` when WAL encryption is enabled. Using `pg_basebackup -F t -X stream` will create a broken replica. -```sql -pg_basebackup: warning: the source has WAL keys, but no WAL encryption configured for the target backups -pg_basebackup: detail: This may lead to exposed data and broken backup -pg_basebackup: hint: Run pg_basebackup with -E to encrypt streamed WAL -``` +Streaming mode (`-X stream`): -You can safely ignore these warnings when using `-X none` or `-X fetch`, since in both cases WAL is not streamed. +* **Must** specify `-E` (`--encrypt-wal`). +* Without `-E`, backups may contain decrypted WAL while `wal_encryption=on` remains in `postgresql.conf` and `pg_tde/wal_keys` are copied. This leads to **startup failures and compromised data in the backup**. + +Fetch mode (`-X fetch`): + +* Compatible with encrypted WAL without requiring any additional flags. + +None (`-X none`): + +* Excludes WAL from the backup and is unaffected by WAL encryption. + + +!!! note + If the source server has `pg_tde/wal_keys`, running `pg_basebackup` with `-X none` or `-X fetch` produces warnings such as: + + ```sql + pg_basebackup: warning: the source has WAL keys, but no WAL encryption configured for the target backups + pg_basebackup: detail: This may lead to exposed data and broken backup + pg_basebackup: hint: Run pg_basebackup with -E to encrypt streamed WAL + ``` + + You can safely ignore the warnings with `-X none` or `-X fetch`, since no WAL streaming occurs. From fb543801dc978f61bb39bfbaadf1990751f0a5ec Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Tue, 26 Aug 2025 11:19:29 +0300 Subject: [PATCH 08/18] Add WAL release note for 2.0 release (#482) - add new date variable for 2.0 release - populated with feedback after code freeze and team comments --- .../docs/release-notes/release-notes-v1.0.md | 6 +- .../docs/release-notes/release-notes-v2.0.md | 74 +++++++++++++++++++ .../docs/release-notes/release-notes.md | 1 + .../docs/templates/pdf_cover_page.tpl | 2 +- contrib/pg_tde/documentation/mkdocs.yml | 8 +- contrib/pg_tde/documentation/variables.yml | 7 +- 6 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 contrib/pg_tde/documentation/docs/release-notes/release-notes-v2.0.md diff --git a/contrib/pg_tde/documentation/docs/release-notes/release-notes-v1.0.md b/contrib/pg_tde/documentation/docs/release-notes/release-notes-v1.0.md index e5035a8a1d6..8a62726818c 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/release-notes-v1.0.md +++ b/contrib/pg_tde/documentation/docs/release-notes/release-notes-v1.0.md @@ -1,6 +1,6 @@ # pg_tde 1.0 ({{date.GA10}}) -The `pg_tde` by Percona extension brings in [Transparent Data Encryption (TDE) :octicons-link-external-16:](../index/index.md) to PostgreSQL and enables you to keep sensitive data safe and secure. +The `pg_tde` by Percona extension brings in [Transparent Data Encryption (TDE) :octicons-link-external-16:](../index/about-tde.md) to PostgreSQL and enables you to keep sensitive data safe and secure. [Get Started](../install.md){.md-button} @@ -8,7 +8,7 @@ The `pg_tde` by Percona extension brings in [Transparent Data Encryption (TDE) : * **`pg_tde` 1.0 is now GA (Generally Available)** -And **stable** for encrypting relational data in PostgreSQL using [Transparent Data Encryption (TDE) :octicons-link-external-16:](../index/index.md). This milestone brings production-level data protection to PostgreSQL workloads. +And **stable** for encrypting relational data in PostgreSQL using [Transparent Data Encryption (TDE) :octicons-link-external-16:](../index/about-tde.md). This milestone brings production-level data protection to PostgreSQL workloads. * **WAL encryption is still in Beta** @@ -16,7 +16,7 @@ The WAL encryption feature is currently still in beta and is not effective unles ## Upgrade considerations -`pg_tde` ({{tdeversion}}) is **not** backward compatible with previous `pg_tde` versions, like Release Candidate 2, due to significant changes in code. This means you **cannot** directly upgrade from one version to another. You must do **a clean installation** of `pg_tde`. +`pg_tde` 1.0 is **not** backward compatible with previous `pg_tde` versions, like Release Candidate 2, due to significant changes in code. This means you **cannot** directly upgrade from one version to another. You must do **a clean installation** of `pg_tde`. ## Known issues diff --git a/contrib/pg_tde/documentation/docs/release-notes/release-notes-v2.0.md b/contrib/pg_tde/documentation/docs/release-notes/release-notes-v2.0.md new file mode 100644 index 00000000000..b70e1765ec1 --- /dev/null +++ b/contrib/pg_tde/documentation/docs/release-notes/release-notes-v2.0.md @@ -0,0 +1,74 @@ +# pg_tde 2.0 ({{date.GA20}}) + +The `pg_tde` by Percona extension brings [Transparent Data Encryption (TDE)](../index/about-tde.md) to PostgreSQL and enables you to keep sensitive data safe and secure. + +[Get Started](../install.md){.md-button} + +## Release Highlights + +### WAL encryption is now generally available + +The WAL (Write-Ahead Logging) encryption feature is now fully supported and production-ready, it adds secure write-ahead logging to `pg_tde`, expanding Percona's PostgreSQL encryption coverage by enabling secure, transparent encryption of write-ahead logs using the same key infrastructure as data encryption. + +### WAL encryption upgrade limitation + +Clusters that used WAL encryption in the beta release (`pg_tde` 1.0 or older) cannot be upgraded to `pg_tde` 2.0. The following error indicates that WAL encryption was enabled: + +```sql +FATAL: principal key not configured +HINT: Use pg_tde_set_server_key_using_global_key_provider() to configure one. +``` + +Clusters that did not use WAL encryption in beta can be upgraded normally. + +### Documentation updates + +* Updated the [Limitations](../index/tde-limitations.md) topic, it now includes WAL encryption limitations and both supported and unsupported WAL tools +* [PG-1858 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1858) - Added a new topic for [Backup with WAL encryption enabled](../how-to/backup-wal-enabled.md) that includes restoring a backup created with WAL encryption +* [PG-1832 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1858) - Added documentation for using the `pg_tde_archive_decrypt` and `pg_tde_restore_encrypt` utilities. These tools are now covered in [CLI Tools](../command-line-tools/cli-tools.md) to guide users on how to archive and restore encrypted WAL segments securely +* [PG-1740 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1740) - Updated documentation for [uninstalling `pg_tde`](../how-to/uninstall.md) with WAL encryption enabled and improved the uninstall instructions to cover cases where TDE is disabled while WAL encryption remains active + +## Known issues + +* The default `mlock` limit on Rocky Linux 8 for ARM64-based architectures equals the memory page size and is 64 Kb. This results in the child process with `pg_tde` failing to allocate another memory page because the max memory limit is reached by the parent process. + +To prevent this, you can change the `mlock` limit to be at least twice bigger than the memory page size: + +* temporarily for the current session using the `ulimit -l ` command. +* set a new hard limit in the `/etc/security/limits.conf` file. To do so, you require the superuser privileges. + +Adjust the limits with caution since it affects other processes running in your system. + +## Changelog + +### New Features + +* [PG-1497 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1497) WAL encryption is now generally available (GA) +* [PG-1037 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1037) - Added support for `pg_rewind` with encrypted WAL +* [PG-1411 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1497) - Added support for `pg_resetwal` with encrypted WAL +* [PG-1603 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1603) - Added support for `pg_basebackup` with encrypted WAL +* [PG-1710 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1710) - Added support for WAL archiving with encrypted WAL +* [PG-1711 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1711) - Added support for incremental backups with encrypted WAL, compatibility has been verified with `pg_combinebackup` and the WAL summarizer tool. +* [PG-1712 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1712) - Added support for `pg_createsubscriber` with encrypted WAL +* [PG-1833 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1833) - Added verified support for using `pg_waldump` with encrypted WAL +* [PG-1834 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1834) - Verified `pg_upgrade` with encryption + +### Improvements + +* [PG-1661 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1661) - Added validation for key material received from providers +* [PG-1667 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1667) - Validated Vault keyring engine type + +### Bugs Fixed + +* [PG-1391 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1391) - Fixed unencrypted checkpoint segment on replica with encrypted key +* [PG-1412 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1412) – Fixed an issue where `XLogFileCopy` failed with encrypted WAL during PITR and `pg_rewind` +* [PG-1452 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1452) - Fixed an issue where `pg_tde_change_key_provider` did not work without the `-D` flag even if `PGDATA` was set +* [PG-1485 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1485) - Fixed an issue where streaming replication failed with an invalid magic number in WAL when `wal_encryption` was enabled +* [PG-1604 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1604) - Fixed a crash during standby promotion caused by an invalid magic number when replaying two-phase transactions from WAL +* [PG-1658 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1658) - Fixed an issue where the global key provider could not be deleted after server restart +* [PG-1835 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1835) - Fixed an issue where `pg_resetwal` corrupted encrypted WAL, causing PostgreSQL to fail at startup with an invalid checkpoint +* [PG-1842 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1842) - Fixed a delay in replica startup with encrypted tables in streaming replication setups +* [PG-1843 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1843) - Fixed performance issues when creating encrypted tables +* [PG-1863 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1863) - Fixed an issue where unnecessary WAL was generated when creating temporary tables +* [PG-1866 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1866) - Fixed an issue where automatic restart after crash sometimes failed with WAL encryption enabled +* [PG-1867 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1867) - Fixed archive recovery with encrypted WAL diff --git a/contrib/pg_tde/documentation/docs/release-notes/release-notes.md b/contrib/pg_tde/documentation/docs/release-notes/release-notes.md index 38d8536e2ba..9f50fcba58c 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/release-notes.md +++ b/contrib/pg_tde/documentation/docs/release-notes/release-notes.md @@ -4,6 +4,7 @@ This page lists all release notes for `pg_tde`, organized by year and version. U ## 2025 +* [2.0](release-notes-v2.0.md) ({{date.GA20}}) * [1.0](release-notes-v1.0.md) ({{date.GA10}}) * [Release Candidate 2 (RC2)](rc2.md) ({{date.RC2}}) * [Release Candidate 1 (RC1)](rc.md) ({{date.RC}}) diff --git a/contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl b/contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl index 9f4fb22ba71..28d0a506f1b 100644 --- a/contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl +++ b/contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl @@ -7,5 +7,5 @@ {% if config.site_description %}

{{ config.site_description }}

{% endif %} -

1.0 (2025-06-30)

+

2.0 (2025-08-30)

\ No newline at end of file diff --git a/contrib/pg_tde/documentation/mkdocs.yml b/contrib/pg_tde/documentation/mkdocs.yml index 5ffd25e3236..bdb43ccb5ed 100644 --- a/contrib/pg_tde/documentation/mkdocs.yml +++ b/contrib/pg_tde/documentation/mkdocs.yml @@ -209,10 +209,10 @@ nav: - faq.md - "Release notes": - "Release notes index": release-notes/release-notes.md - - "2025": - - "1.0": release-notes/release-notes-v1.0.md - - "Release Candidate 2": release-notes/rc2.md - - "Release Candidate 1": release-notes/rc.md + - "2.0": release-notes/release-notes-v2.0.md + - "1.0": release-notes/release-notes-v1.0.md + - "Release Candidate 2": release-notes/rc2.md + - "Release Candidate 1": release-notes/rc.md - "2024 (Alpha 1 - Beta 2)": - "Beta 2": release-notes/beta2.md - "Beta": release-notes/beta.md diff --git a/contrib/pg_tde/documentation/variables.yml b/contrib/pg_tde/documentation/variables.yml index bf698a07c22..a61f6cb0887 100644 --- a/contrib/pg_tde/documentation/variables.yml +++ b/contrib/pg_tde/documentation/variables.yml @@ -1,12 +1,13 @@ #Variables used throughout the docs -latestreleasenotes: 'release-notes-v1.0' -tdeversion: '1.0' -release: '1.0' +latestreleasenotes: 'release-notes-v2.0' +tdeversion: '2.0' +release: '2.0' pgversion17: '17.5' tdebranch: release-17.5.2 date: + GA20: '2025-08-30' GA10: '2025-06-30' RC2: '2025-05-29' RC: '2025-03-27' From a711d8befab9f99cf5f14fb930eb7170360ea489 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Mon, 25 Aug 2025 20:42:29 +0300 Subject: [PATCH 09/18] Fix possible _keys file loss during key rotation There is no reason to do durable_unlink before durable_rename. Rename can handle existing file. But with this sequence, the cluster may endup in unrecoverable state should server crash in-between this two ops, as there is going to be no "_keys" at all. The current sequence may also cause an issue the backup: , , . And no "_keys" file in the backup as the result. --- contrib/pg_tde/src/access/pg_tde_tdemap.c | 3 +-- contrib/pg_tde/src/access/pg_tde_xlog_keys.c | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/contrib/pg_tde/src/access/pg_tde_tdemap.c b/contrib/pg_tde/src/access/pg_tde_tdemap.c index 1a9e118771c..082a9ad8f2f 100644 --- a/contrib/pg_tde/src/access/pg_tde_tdemap.c +++ b/contrib/pg_tde/src/access/pg_tde_tdemap.c @@ -297,10 +297,9 @@ pg_tde_perform_rotate_key(const TDEPrincipalKey *principal_key, const TDEPrincip CloseTransientFile(new_fd); /* - * Do the final steps - replace the current _map with the file with new + * Do the final step - replace the current _keys with the file with new * data */ - durable_unlink(old_path, ERROR); durable_rename(new_path, old_path, ERROR); /* diff --git a/contrib/pg_tde/src/access/pg_tde_xlog_keys.c b/contrib/pg_tde/src/access/pg_tde_xlog_keys.c index 4c14625e008..f1c64aa5a32 100644 --- a/contrib/pg_tde/src/access/pg_tde_xlog_keys.c +++ b/contrib/pg_tde/src/access/pg_tde_xlog_keys.c @@ -718,10 +718,9 @@ pg_tde_perform_rotate_server_key(const TDEPrincipalKey *principal_key, CloseTransientFile(new_fd); /* - * Do the final steps - replace the current WAL key file with the file - * with new data. + * Do the final step - replace the current WAL key file with the file with + * new data. */ - durable_unlink(get_wal_key_file_path(), ERROR); durable_rename(tmp_path, get_wal_key_file_path(), ERROR); /* From 2364be29cc8ae5b538adcb2bc2d87870e2b70ecb Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Tue, 26 Aug 2025 16:37:35 +0300 Subject: [PATCH 10/18] Fix XLogging of rotated key Before this commit, we XLogged the provider ID (keyringId) of the old key. Yet, we then attempt to fetch the new key from the old provider during the Redo, which obviously fails and crashes the recovery. So the next steps lead to the recovery stalemate: - Create new provider (with new destination - mount_path, url etc). - Create new server/global key. - Rotate key. - This commit fixes it by Xlogging the new key's provider ID. For: PG-1895 --- contrib/pg_tde/src/access/pg_tde_tdemap.c | 4 ++-- contrib/pg_tde/src/access/pg_tde_xlog_keys.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/pg_tde/src/access/pg_tde_tdemap.c b/contrib/pg_tde/src/access/pg_tde_tdemap.c index 082a9ad8f2f..1a910f19d39 100644 --- a/contrib/pg_tde/src/access/pg_tde_tdemap.c +++ b/contrib/pg_tde/src/access/pg_tde_tdemap.c @@ -317,8 +317,8 @@ pg_tde_perform_rotate_key(const TDEPrincipalKey *principal_key, const TDEPrincip { XLogPrincipalKeyRotate xlrec; - xlrec.databaseId = principal_key->keyInfo.databaseId; - xlrec.keyringId = principal_key->keyInfo.keyringId; + xlrec.databaseId = new_principal_key->keyInfo.databaseId; + xlrec.keyringId = new_principal_key->keyInfo.keyringId; memcpy(xlrec.keyName, new_principal_key->keyInfo.name, sizeof(new_principal_key->keyInfo.name)); XLogBeginInsert(); diff --git a/contrib/pg_tde/src/access/pg_tde_xlog_keys.c b/contrib/pg_tde/src/access/pg_tde_xlog_keys.c index f1c64aa5a32..fc558b84798 100644 --- a/contrib/pg_tde/src/access/pg_tde_xlog_keys.c +++ b/contrib/pg_tde/src/access/pg_tde_xlog_keys.c @@ -738,8 +738,8 @@ pg_tde_perform_rotate_server_key(const TDEPrincipalKey *principal_key, { XLogPrincipalKeyRotate xlrec; - xlrec.databaseId = principal_key->keyInfo.databaseId; - xlrec.keyringId = principal_key->keyInfo.keyringId; + xlrec.databaseId = new_principal_key->keyInfo.databaseId; + xlrec.keyringId = new_principal_key->keyInfo.keyringId; memcpy(xlrec.keyName, new_principal_key->keyInfo.name, sizeof(new_principal_key->keyInfo.name)); XLogBeginInsert(); From 532d264054184751dd3cdb24c28f0e633643fe3f Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Thu, 28 Aug 2025 09:29:51 +0300 Subject: [PATCH 11/18] Remove an extra s from param and remove ensure_new_key param (#555) --- contrib/pg_tde/documentation/docs/functions.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index 82e6e3449a2..2452674a670 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -229,7 +229,7 @@ Use these functions to create a new principal key at a given keyprover, and to u Principal keys are stored on key providers by the name specified in this function - for example, when using the Vault provider, after creating a key named "foo", a key named "foo" will be visible on the Vault server at the specified mount point. -### pg_tde_creates_key_using_database_key_provider +### pg_tde_create_key_using_database_key_provider Creates a principal key using the database-local key provider with the specified name. Use this key later with [`pg_tde_set_key_using_database_key_provider()`](#pg_tde_set_key_using_database_key_provider). @@ -286,12 +286,6 @@ SELECT pg_tde_set_server_key_using_global_key_provider( ); ``` -The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: - -* If set to `true`, a new key must be unique. - If the provider already stores a key by that name, the function returns an error. -* If set to `false` (default), an existing principal key may be reused. - ### pg_tde_set_default_key_using_global_key_provider Sets or rotates the default principal key for the server using the specified global key provider. From afdbffb422bf3e5f28d988ae86ce16b16600924e Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Thu, 28 Aug 2025 10:31:44 +0300 Subject: [PATCH 12/18] Add information regarding key rotation during backups for pg_basebackup making servers fail to start (#550) - add as known issue in release notes - fix a broken link in features.md (not related to issue...) - add to global key providers a warning about keyring provider with WAL encrypt - add new subtopic in Backup WAL about key rotations during backups for file-based key providers Based on PG-1895 description. --- contrib/pg_tde/documentation/docs/features.md | 2 +- .../overview.md | 3 +++ .../docs/how-to/backup-wal-enabled.md | 16 ++++++++++++++++ .../docs/release-notes/release-notes-v2.0.md | 16 ++++++++++++---- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/features.md b/contrib/pg_tde/documentation/docs/features.md index 3f51c25acea..1db6fc24f09 100644 --- a/contrib/pg_tde/documentation/docs/features.md +++ b/contrib/pg_tde/documentation/docs/features.md @@ -17,7 +17,7 @@ The following features are available for the extension: * Single-tenancy support via a [global keyring provider](global-key-provider-configuration/set-principal-key.md) * [Multi-tenancy support](how-to/multi-tenant-setup.md) * Table-level granularity for encryption and access control -* Multiple [Key management options](global-key-provider-configuration/index.md) +* Multiple [Key management options](global-key-provider-configuration/overview.md) ## Next steps diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/overview.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/overview.md index 14aa13e569c..a6c7e4703fd 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/overview.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/overview.md @@ -11,6 +11,9 @@ To use an external KMS with `pg_tde`, follow these two steps: !!! note While key files may be acceptable for **local** or **testing environments**, KMS integration is the recommended approach for production deployments. +!!! warning + Do not rotate encryption keys while `pg_basebackup` is running. Standbys or standalone clusters created from such backups may fail to start during WAL replay. Schedule rotations outside your backup windows and run a new full backup afterward. + `pg_tde` has been tested with the following key providers: | KMS Provider | Description | Documentation | diff --git a/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md b/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md index a3214d9a6a3..97c2804b40f 100644 --- a/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md +++ b/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md @@ -21,6 +21,22 @@ Also copy any external files referenced by your providers configuration (such as - The `-E` flag only works with the `-X stream` option (default). It is not compatible with `-X none` or `-X fetch`. For more information, see [the other WAL methods topic](#other-wal-methods). - The `-E` flag is only supported with the plain output format (`-F p`). It cannot be used with the tar output format (`-F t`). +## Key rotation during backups + +!!! warning + Do not rotate SMGR or WAL encryption keys while `pg_basebackup` is running. Standbys or standalone clusters created from such backups may fail to start during WAL replay. + +Rotations during a base backup can leave the standby in an inconsistent state where it cannot retrieve the correct key history. + +For example, you may see errors such as: + +```sql +FATAL: failed to retrieve principal key "database_keyXXXX" from key provider "providerYYYY" +CONTEXT: WAL redo at ... ROTATE_PRINCIPAL_KEY ... +``` + +To ensure standby recoverability, plan key rotations outside backup windows or take a new full backup after rotation completes. + ## Restore a backup created with WAL encryption When you want to restore a backup created with `pg_basebackup -E`: diff --git a/contrib/pg_tde/documentation/docs/release-notes/release-notes-v2.0.md b/contrib/pg_tde/documentation/docs/release-notes/release-notes-v2.0.md index b70e1765ec1..22c74ba52d3 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/release-notes-v2.0.md +++ b/contrib/pg_tde/documentation/docs/release-notes/release-notes-v2.0.md @@ -30,14 +30,22 @@ Clusters that did not use WAL encryption in beta can be upgraded normally. ## Known issues +* Rotating encryption keys while `pg_basebackup` is running may cause standbys or standalone clusters initialized from the backup to fail during WAL replay. + + Avoid key rotations during backups. Run a new full backup after completing a rotation. + +* Using `pg_basebackup` with `--wal-method=fetch` produces warnings. + + This behavior is expected and will be addressed in a future release. + * The default `mlock` limit on Rocky Linux 8 for ARM64-based architectures equals the memory page size and is 64 Kb. This results in the child process with `pg_tde` failing to allocate another memory page because the max memory limit is reached by the parent process. -To prevent this, you can change the `mlock` limit to be at least twice bigger than the memory page size: + To prevent this, you can change the `mlock` limit to be at least twice bigger than the memory page size: -* temporarily for the current session using the `ulimit -l ` command. -* set a new hard limit in the `/etc/security/limits.conf` file. To do so, you require the superuser privileges. + * temporarily for the current session using the `ulimit -l ` command. + * set a new hard limit in the `/etc/security/limits.conf` file. To do so, you require the superuser privileges. -Adjust the limits with caution since it affects other processes running in your system. + Adjust the limits with caution since it affects other processes running in your system. ## Changelog From 89f5000235e0e260e8c2fbf1a42825cf5b0bd11b Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Thu, 28 Aug 2025 15:13:19 +0300 Subject: [PATCH 13/18] Update Patroni config file (#551) The original config file was taken from [here](https://github.com/jobinau/pgscripts/blob/main/patroni/patroni.yml) and it is currently replaced with a more "up to date" version [here](https://github.com/Percona-Lab/pg_tde-demo-cluster/blob/bbb1df011f79667e7d8960191a775da90574e2c5/templates/patroni.yml.j2). --- .../docs/index/tde-limitations.md | 62 +++++++++++-------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/index/tde-limitations.md b/contrib/pg_tde/documentation/docs/index/tde-limitations.md index 5b4e4328722..fb9e5de0b49 100644 --- a/contrib/pg_tde/documentation/docs/index/tde-limitations.md +++ b/contrib/pg_tde/documentation/docs/index/tde-limitations.md @@ -34,17 +34,13 @@ The following is a Percona-tested example configuration. ```yaml # Example Patroni configuration file maintained by Percona # Source: https://github.com/jobinau/pgscripts/blob/main/patroni/patroni.yml - scope: postgres - namespace: /db/ - name: postgresql0 - + scope: tde + name: pg1 restapi: listen: 0.0.0.0:8008 - connect_address: 127.0.0.1:8008 - - etcd: - host: 127.0.0.1:2379 - + connect_address: pg1:8008 + etcd3: + host: etcd1:2379 bootstrap: dcs: ttl: 30 @@ -55,27 +51,29 @@ The following is a Percona-tested example configuration. use_pg_rewind: true use_slots: true parameters: - max_connections: 100 - shared_buffers: 1GB - wal_level: replica - hot_standby: "on" - wal_keep_size: 256MB - max_wal_senders: 10 - max_replication_slots: 10 - + archive_command: "/lib/postgresql/17/bin/pg_tde_archive_decrypt %f %p \"pgbackrest --stanza=tde archive-push %%p\"" + archive_timeout: 600s + archive_mode: "on" + logging_collector: "on" + restore_command: "/lib/postgresql/17/bin/pg_tde_restore_encrypt %f %p \"pgbackrest --stanza=tde archive-get %%f \\\"%%p\\\"\"" + pg_hba: + - local all all peer + - host all all 0.0.0.0/0 scram-sha-256 + - host all all ::/0 scram-sha-256 + - local replication all peer + - host replication all 0.0.0.0/0 scram-sha-256 + - host replication all ::/0 scram-sha-256 initdb: - - encoding: UTF8 - - data-checksums - - pg_hba: - - host replication replicator 127.0.0.1/32 md5 - - host all all 0.0.0.0/0 md5 - + - encoding: UTF8 + - data-checksums + - set: shared_preload_libraries=pg_tde + post_init: /usr/local/bin/setup_cluster.sh postgresql: listen: 0.0.0.0:5432 - connect_address: 127.0.0.1:5432 - data_dir: /var/lib/postgresql/data - bin_dir: /usr/lib/postgresql/14/bin + connect_address: pg1:5432 + data_dir: /var/lib/postgresql/patroni-17 + bin_dir: /lib/postgresql/17/bin + pgpass: /var/lib/postgresql/patronipass authentication: replication: username: replicator @@ -83,6 +81,16 @@ The following is a Percona-tested example configuration. superuser: username: postgres password: secretpassword + parameters: + unix_socket_directories: /tmp + # Use unix_socket_directories: /var/run/postgresql for Debian/Ubuntu distributions + watchdog: + mode: off + tags: + nofailover: false + noloadbalance: false + clonefrom: false + nosync: false ``` !!! warning From a3f36c89837c0881ee6813755325a1dd15c68b19 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Thu, 28 Aug 2025 15:37:16 +0300 Subject: [PATCH 14/18] Add pg_receivewal to list of unsupported tools with WAL encrypt (#556) --- contrib/pg_tde/documentation/docs/index/tde-limitations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/pg_tde/documentation/docs/index/tde-limitations.md b/contrib/pg_tde/documentation/docs/index/tde-limitations.md index fb9e5de0b49..3d0e95077b5 100644 --- a/contrib/pg_tde/documentation/docs/index/tde-limitations.md +++ b/contrib/pg_tde/documentation/docs/index/tde-limitations.md @@ -10,6 +10,7 @@ Limitations of `pg_tde` {{release}}: The following tools are currently unsupported with `pg_tde` WAL encryption: * `pg_createsubscriber` +* `pg_receivewal` * `pg_verifybackup` (checksum mismatch with encrypted WAL) The following tools and extensions in Percona Distribution for PostgreSQL have been tested and verified to work with `pg_tde` WAL encryption: From d48db2b58f8dd4507dbfc9ced98e4a4175ff03c8 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Fri, 29 Aug 2025 09:28:50 +0300 Subject: [PATCH 15/18] Update warnings for basebackup (#558) --- .../pg_tde/documentation/docs/how-to/backup-wal-enabled.md | 4 ++-- .../documentation/docs/release-notes/release-notes-v2.0.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md b/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md index 97c2804b40f..7b51bbbd176 100644 --- a/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md +++ b/contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md @@ -24,9 +24,9 @@ Also copy any external files referenced by your providers configuration (such as ## Key rotation during backups !!! warning - Do not rotate SMGR or WAL encryption keys while `pg_basebackup` is running. Standbys or standalone clusters created from such backups may fail to start during WAL replay. + Do not create, change, or rotate global key providers (or their keys) while `pg_basebackup` is running. Standbys or standalone clusters created from such backups may fail to start during WAL replay and may also lead to the corruption of encrypted data (tables, indexes, and other relations). -Rotations during a base backup can leave the standby in an inconsistent state where it cannot retrieve the correct key history. +Creating, changing, or rotating global key providers (or their keys) during a base backup can leave the standby in an inconsistent state where it cannot retrieve the correct key history. For example, you may see errors such as: diff --git a/contrib/pg_tde/documentation/docs/release-notes/release-notes-v2.0.md b/contrib/pg_tde/documentation/docs/release-notes/release-notes-v2.0.md index 22c74ba52d3..5a11ed3300d 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/release-notes-v2.0.md +++ b/contrib/pg_tde/documentation/docs/release-notes/release-notes-v2.0.md @@ -30,9 +30,9 @@ Clusters that did not use WAL encryption in beta can be upgraded normally. ## Known issues -* Rotating encryption keys while `pg_basebackup` is running may cause standbys or standalone clusters initialized from the backup to fail during WAL replay. +* Creating, changing, or rotating global key providers (or their keys) while `pg_basebackup` is running may cause standbys or standalone clusters initialized from the backup to fail during WAL replay and may also lead to the corruption of encrypted data (tables, indexes, and other relations). - Avoid key rotations during backups. Run a new full backup after completing a rotation. + Avoid making these actions during backup windows. Run a new full backup after completing a rotation or provider update. * Using `pg_basebackup` with `--wal-method=fetch` produces warnings. From 7ef4fceabfe3c273d14f83dce35feb4ce2cb6651 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Sat, 30 Aug 2025 08:43:01 +0300 Subject: [PATCH 16/18] Update `pg_verifybackup` tool with workaround (#553) --- contrib/pg_tde/documentation/docs/index/tde-limitations.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/pg_tde/documentation/docs/index/tde-limitations.md b/contrib/pg_tde/documentation/docs/index/tde-limitations.md index 3d0e95077b5..905caa04470 100644 --- a/contrib/pg_tde/documentation/docs/index/tde-limitations.md +++ b/contrib/pg_tde/documentation/docs/index/tde-limitations.md @@ -11,7 +11,8 @@ The following tools are currently unsupported with `pg_tde` WAL encryption: * `pg_createsubscriber` * `pg_receivewal` -* `pg_verifybackup` (checksum mismatch with encrypted WAL) +* `pg_verifybackup` by default fails with checksum or WAL key size mismatch errors. + As a workaround, use `-s` (skip checksum) and `-n` (`--no-parse-wal`) to verify backups. The following tools and extensions in Percona Distribution for PostgreSQL have been tested and verified to work with `pg_tde` WAL encryption: From c4c685160cb8990ecd0c5d05bb05cf76b9e9addc Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Mon, 1 Sep 2025 17:41:18 +0300 Subject: [PATCH 17/18] Small date fix (#568) --- contrib/pg_tde/documentation/variables.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pg_tde/documentation/variables.yml b/contrib/pg_tde/documentation/variables.yml index a61f6cb0887..2ff670155b9 100644 --- a/contrib/pg_tde/documentation/variables.yml +++ b/contrib/pg_tde/documentation/variables.yml @@ -7,7 +7,7 @@ pgversion17: '17.5' tdebranch: release-17.5.2 date: - GA20: '2025-08-30' + GA20: '2025-09-01' GA10: '2025-06-30' RC2: '2025-05-29' RC: '2025-03-27' From f16d4e5dc8ea340facdbf43535058755dc6dbc47 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Mon, 1 Sep 2025 17:55:52 +0300 Subject: [PATCH 18/18] Update git ver and pdf cover date (#569) --- contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl | 2 +- contrib/pg_tde/documentation/variables.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl b/contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl index 28d0a506f1b..576e8d262ef 100644 --- a/contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl +++ b/contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl @@ -7,5 +7,5 @@ {% if config.site_description %}

{{ config.site_description }}

{% endif %} -

2.0 (2025-08-30)

+

2.0 (2025-09-01)

\ No newline at end of file diff --git a/contrib/pg_tde/documentation/variables.yml b/contrib/pg_tde/documentation/variables.yml index 2ff670155b9..1427a440653 100644 --- a/contrib/pg_tde/documentation/variables.yml +++ b/contrib/pg_tde/documentation/variables.yml @@ -4,7 +4,7 @@ latestreleasenotes: 'release-notes-v2.0' tdeversion: '2.0' release: '2.0' pgversion17: '17.5' -tdebranch: release-17.5.2 +tdebranch: release-17.5.3 date: GA20: '2025-09-01'