Merge pull request #571 from percona/release-17.5.3

Merge back release 17.5.3
pull/238/head
Artem Gavrilov 1 week ago committed by GitHub
commit d0b53f35fb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 2
      ci_scripts/env.sh
  2. 2
      configure
  3. 2
      configure.ac
  4. 2
      contrib/pg_tde/Makefile
  5. 5
      contrib/pg_tde/documentation/docs/architecture/architecture.md
  6. 19
      contrib/pg_tde/documentation/docs/command-line-tools/cli-tools.md
  7. 57
      contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-archive-decrypt.md
  8. 52
      contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-restore-encrypt.md
  9. 3
      contrib/pg_tde/documentation/docs/command-line-tools/pg-waldump.md
  10. 5
      contrib/pg_tde/documentation/docs/faq.md
  11. 8
      contrib/pg_tde/documentation/docs/features.md
  12. 13
      contrib/pg_tde/documentation/docs/functions.md
  13. 18
      contrib/pg_tde/documentation/docs/global-key-provider-configuration/overview.md
  14. 12
      contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md
  15. 2
      contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md
  16. 77
      contrib/pg_tde/documentation/docs/how-to/backup-wal-enabled.md
  17. 10
      contrib/pg_tde/documentation/docs/index/how-does-tde-work.md
  18. 107
      contrib/pg_tde/documentation/docs/index/tde-limitations.md
  19. 6
      contrib/pg_tde/documentation/docs/release-notes/release-notes-v1.0.md
  20. 82
      contrib/pg_tde/documentation/docs/release-notes/release-notes-v2.0.md
  21. 1
      contrib/pg_tde/documentation/docs/release-notes/release-notes.md
  22. 2
      contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl
  23. 2
      contrib/pg_tde/documentation/docs/test.md
  24. 7
      contrib/pg_tde/documentation/docs/wal-encryption.md
  25. 17
      contrib/pg_tde/documentation/mkdocs.yml
  26. 9
      contrib/pg_tde/documentation/variables.yml
  27. 2
      contrib/pg_tde/expected/version.out
  28. 1
      contrib/pg_tde/meson.build
  29. 4
      contrib/pg_tde/pg_tde--1.0--2.0.sql
  30. 2
      contrib/pg_tde/pg_tde.control
  31. 7
      contrib/pg_tde/src/access/pg_tde_tdemap.c
  32. 9
      contrib/pg_tde/src/access/pg_tde_xlog_keys.c
  33. 2
      contrib/pg_tde/src/include/pg_tde.h
  34. 2
      contrib/pg_tde/t/expected/basic.out
  35. 2
      meson.build
  36. 2
      src/test/modules/test_misc/t/008_percona_server_version.pl

@ -1,3 +1,3 @@
#!/bin/bash
export PERCONA_SERVER_VERSION=17.5.2
export PERCONA_SERVER_VERSION=17.5.3

2
configure vendored

@ -2856,7 +2856,7 @@ cat >>confdefs.h <<_ACEOF
_ACEOF
PG_PERCONAVERSION=2
PG_PERCONAVERSION=3
cat >>confdefs.h <<_ACEOF
#define PG_PERCONAVERSION "$PG_PERCONAVERSION"

@ -37,7 +37,7 @@ AC_DEFINE_UNQUOTED(PG_MAJORVERSION, "$PG_MAJORVERSION", [PostgreSQL major versio
AC_DEFINE_UNQUOTED(PG_MAJORVERSION_NUM, $PG_MAJORVERSION, [PostgreSQL major version number])
AC_DEFINE_UNQUOTED(PG_MINORVERSION_NUM, $PG_MINORVERSION, [PostgreSQL minor version number])
[PG_PERCONAVERSION=2]
[PG_PERCONAVERSION=3]
AC_DEFINE_UNQUOTED(PG_PERCONAVERSION, "$PG_PERCONAVERSION", [PostgreSQL Percona version as a string])
PGAC_ARG_REQ(with, extra-version, [STRING], [append STRING to version],

@ -1,7 +1,7 @@
PGFILEDESC = "pg_tde access method"
MODULE_big = pg_tde
EXTENSION = pg_tde
DATA = pg_tde--1.0.sql
DATA = pg_tde--1.0--2.0.sql pg_tde--1.0.sql
# Since meson supports skipping test suites this is a make only feature
ifndef TDE_MODE

@ -20,7 +20,7 @@ The following sections break down the key architectural components of this desig
* Indexes
* Sequences
* Temporary tables
* Write Ahead Log (WAL), still in beta. **Do not enable this feature in production environments**.
* Write Ahead Log (WAL)
**Extension** means that `pg_tde` should be implemented only as an extension, possibly compatible with any PostgreSQL distribution, including the open source community version. This requires changes in the PostgreSQL core to make it more extensible. Therefore, `pg_tde` currently works only with the [Percona Server for PostgreSQL](https://docs.percona.com/postgresql/17/index.html) - a binary replacement of community PostgreSQL and included in Percona Distribution for PostgreSQL.
@ -82,9 +82,6 @@ Later decisions are made using a slightly modified Storage Manager (SMGR) API: w
### WAL encryption
!!! note
The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**.
WAL encryption is controlled globally via a global GUC variable, `pg_tde.wal_encrypt`, that requires a server restart.
WAL keys also contain the [LSN](https://www.postgresql.org/docs/17/wal-internals.html) of the first WAL write after key creation. This allows `pg_tde` to know which WAL ranges are encrypted or not and with which key.

@ -1,7 +1,18 @@
# Overview of pg_tde CLI tools
The `pg_tde` extension introduces new command-line utilities and extends some existing PostgreSQL tools to support encrypted WAL and tables. These include:
The `pg_tde` extension introduces new command-line utilities and extends some existing PostgreSQL tools to support encrypted WAL and tables.
* [pg_tde_change_key_provider](../command-line-tools/pg-tde-change-key-provider.md): change encryption key provider for a database
* [pg_waldump](../command-line-tools/pg-waldump.md): inspect and decrypt WAL files
* [pg_checksums](../command-line-tools/pg-tde-checksums.md): verify data checksums (non-encrypted files only)
## New tools
These tools are introduced by `pg_tde` to support key rotation and WAL encryption workflows:
* [pg_tde_change_key_provider](./pg-tde-change-key-provider.md): change the encryption key provider for a database
* [pg_tde_archive_decrypt](./pg-tde-archive-decrypt.md): decrypts WAL before archiving
* [pg_tde_restore_encrypt](./pg-tde-restore-encrypt.md): a custom restore command for making sure the restored WAL is encrypted
## Extended tools
These existing PostgreSQL tools are enhanced to support `pg_tde`:
* [pg_checksums](./pg-tde-checksums.md): verify data checksums (non-encrypted files only)
* [pg_waldump](./pg-waldump.md): inspect and decrypt WAL files

@ -0,0 +1,57 @@
# pg_tde_archive_decrypt
The `pg_tde_archive_decrypt` tool wraps an archive command and decrypts WAL files before archiving. It allows external tools to access unencrypted WAL data, which is required because WAL encryption keys in the two-key hierarchy are host-specific and may not be available on the replay host.
!!! tip
For more information on the encryption architecture and key hierarchy, see [Architecture](../architecture/architecture.md).
This tool is often used in conjunction with [pg_tde_restore_encrypt](./pg-tde-restore-encrypt.md) to support WAL archive.
## How it works
1. Decrypts the WAL segment to a temporary file on a RAM disk (`/dev/shm`)
2. Replaces `%p` and `%f` in the archive command with the path and name of the decrypted file
3. Executes the archive command
!!! note
To ensure security, encrypt the files stored in your WAL archive using tools like `PgBackRest`.
## Usage
```bash
pg_tde_archive_decrypt [OPTION]
pg_tde_archive_decrypt DEST-NAME SOURCE-PATH ARCHIVE-COMMAND
```
## Parameter descriptions
* `DEST-NAME`: name of the WAL file to send to the archive
* `SOURCE-PATH`: path to the original encrypted WAL file
* `ARCHIVE-COMMAND`: archive command to wrap. `%p` and `%f` are replaced with the decrypted WAL file path and WAL file name, respectively.
## Options
* `-V, --version`: show version information, then exit
* `-?, --help`: show help information, then exit
!!! note
Any `%f` or `%p` parameter in `ARCHIVE-COMMAND` has to be escaped as `%%f` or `%%p` respectively if used as `archive_command` in `postgresql.conf`.
## Examples
### Using `cp`
```ini
archive_command='pg_tde_archive_decrypt %f %p "cp %%p /mnt/server/archivedir/%%f"'
```
### Using `PgBackRest`
```ini
archive_command='pg_tde_archive_decrypt %f %p "pgbackrest --stanza=your_stanza archive-push %%p"'
```
!!! warning
When using PgBackRest with WAL encryption, disable PostgreSQL data checksums. Otherwise, PgBackRest may spam error messages, and in some package builds the log statement can cause crashes.

@ -0,0 +1,52 @@
# pg_tde_restore_encrypt
The `pg_tde_restore_encrypt` tool wraps a normal restore command from the WAL archive and writes them to disk in a format compatible with `pg_tde`.
!!! note
This command is often use together with [pg_tde_archive_decrypt](./pg-tde-archive-decrypt.md).
## How it works
1. Replaces `%f` and `%p` in the restore command with the WAL file name and temporary file path (in `/dev/shm`)
2. Runs the restore command to fetch the unencrypted WAL from the archive and write it to the temp file
3. Encrypts the temp file and writes it to the destination path in PostgreSQL’s data directory
## Usage
```bash
pg_tde_restore_encrypt [OPTION]
pg_tde_restore_encrypt SOURCE-NAME DEST-PATH RESTORE-COMMAND
```
## Parameter descriptions
* `SOURCE-NAME`: name of the WAL file to retrieve from the archive
* `DEST-PATH`: path where the encrypted WAL file should be written
* `RESTORE-COMMAND`: restore command to wrap; `%p` and `%f` are replaced with the WAL file name and path to write the unencrypted WAL, respectively
## Options
* `-V, --version`: show version information, then exit
* `-?, --help`: show help information, then exit
!!! note
Any `%f` or `%p` parameter in `RESTORE-COMMAND` has to be escaped as `%%f` or `%%p` respectively if used as `restore_command` in `postgresql.conf`.
## Examples
### Using `cp`
```ini
restore_command='pg_tde_restore_encrypt %f %p "cp /mnt/server/archivedir/%%f %%p"'
```
### Using `PgBackRest`
```ini
restore_command='pg_tde_restore_encrypt %f %p "pgbackrest --stanza=your_stanza archive-get %%f \"%%p\""'
```
!!! warning
When using PgBackRest with WAL encryption, disable PostgreSQL data checksums. Otherwise, PgBackRest may spam error messages, and in some package builds the log statement can cause crashes.

@ -2,9 +2,6 @@
[`pg_waldump` :octicons-link-external-16:](https://www.postgresql.org/docs/current/pgwaldump.html) is a tool to display a human-readable rendering of the Write-Ahead Log (WAL) of a PostgreSQL database cluster.
!!! warning
The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**.
To read encrypted WAL records, `pg_waldump` supports the following additional arguments:
* `keyring_path` is the directory where the keyring configuration files for WAL are stored. The following files are included:

@ -94,9 +94,6 @@ The principal key is used to encrypt the internal keys. The principal key is sto
### WAL encryption
!!! note
WAL encryption is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**.
WAL encryption is done globally for the entire database cluster. All modifications to any database within a PostgreSQL cluster are written to the same WAL to maintain data consistency and integrity and ensure that PostgreSQL cluster can be restored to a consistent state. Therefore, WAL is encrypted globally.
When you turn on WAL encryption, `pg_tde` encrypts entire WAL files starting from the first WAL write after the server was started with the encryption turned on.
@ -140,7 +137,7 @@ Since the `SET ACCESS METHOD` command drops hint bits and this may affect the pe
You must restart the database in the following cases to apply the changes:
* after you enabled the `pg_tde` extension
* when enabling WAL encryption, which is currently in beta. **Do not enable this feature in production environments**.
* when enabling WAL encryption
After that, no database restart is required. When you create or alter the table using the `tde_heap` access method, the files are marked as those that require encryption. The encryption happens at the storage manager level, before a transaction is written to disk. Read more about [how tde_heap works](index/table-access-method.md#how-tde_heap-works-with-pg_tde).

@ -17,6 +17,10 @@ The following features are available for the extension:
* Single-tenancy support via a [global keyring provider](global-key-provider-configuration/set-principal-key.md)
* [Multi-tenancy support](how-to/multi-tenant-setup.md)
* Table-level granularity for encryption and access control
* Multiple [Key management options](global-key-provider-configuration/index.md)
* Multiple [Key management options](global-key-provider-configuration/overview.md)
[Learn more about TDE and pg_tde :material-arrow-right:](index/about-tde.md){.md-button} [Get started with installation :material-arrow-right:](install.md){.md-button}
## Next steps
Learn more about how `pg_tde` implements Transparent Data Encryption:
[About Transparent Data Encryption :material-arrow-right:](index/about-tde.md){.md-button}

@ -229,7 +229,7 @@ Use these functions to create a new principal key at a given keyprover, and to u
Principal keys are stored on key providers by the name specified in this function - for example, when using the Vault provider, after creating a key named "foo", a key named "foo" will be visible on the Vault server at the specified mount point.
### pg_tde_creates_key_using_database_key_provider
### pg_tde_create_key_using_database_key_provider
Creates a principal key using the database-local key provider with the specified name. Use this key later with [`pg_tde_set_key_using_database_key_provider()`](#pg_tde_set_key_using_database_key_provider).
@ -242,7 +242,7 @@ SELECT pg_tde_create_key_using_database_key_provider(
### pg_tde_create_key_using_global_key_provider
Creates a principal key at a global key provider with the given name. Use this key later with the `pg_tde_set_` series of functions.
Creates a principal key at a global key provider with the given name. Use this key later with the `pg_tde_set_*` series of functions.
```sql
SELECT pg_tde_create_key_using_global_key_provider(
@ -286,15 +286,6 @@ SELECT pg_tde_set_server_key_using_global_key_provider(
);
```
!!! warning
The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**.
The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation:
* If set to `true`, a new key must be unique.
If the provider already stores a key by that name, the function returns an error.
* If set to `false` (default), an existing principal key may be reused.
### pg_tde_set_default_key_using_global_key_provider
Sets or rotates the default principal key for the server using the specified global key provider.

@ -9,10 +9,18 @@ To use an external KMS with `pg_tde`, follow these two steps:
2. Set the [Global Principal Key](set-principal-key.md)
!!! note
While keyfiles may be acceptable for **local** or **testing environments**, KMS integration is the recommended approach for production deployments.
While key files may be acceptable for **local** or **testing environments**, KMS integration is the recommended approach for production deployments.
Select your prefered configuration from the links below:
!!! warning
Do not rotate encryption keys while `pg_basebackup` is running. Standbys or standalone clusters created from such backups may fail to start during WAL replay. Schedule rotations outside your backup windows and run a new full backup afterward.
[KMIP Configuration :material-arrow-right:](kmip-server.md){.md-button}
[Vault Configuration :material-arrow-right:](vault.md){.md-button}
[Keyring File Configuration (not recommended) :material-arrow-right:](keyring.md){.md-button}
`pg_tde` has been tested with the following key providers:
| KMS Provider | Description | Documentation |
|--------------------|-------------------------------------------------------|---------------|
| **KMIP** | Standard Key Management Interoperability Protocol. | [Configure KMIP →](kmip-server.md) |
| **Vault** | HashiCorp Vault integration (KV v2 API, KMIP engine). | [Configure Vault →](vault.md) |
| **Fortanix** | Fortanix DSM key management. | [Configure Fortanix →](kmip-fortanix.md) |
| **Thales** | Thales CipherTrust Manager and DSM. | [Configure Thales →](kmip-thales.md) |
| **OpenBao** | Community fork of Vault, supporting KV v2. | [Configure OpenBao →](kmip-openbao.md) |
| **Keyring file** *(not recommended)* | Local key file for dev/test only. | [Configure keyring file →](keyring.md) |

@ -1,10 +1,16 @@
# Global Principal Key configuration
You can configure a default principal key using a global key provider. This key will be used by all databases that do not have their own encryption keys configured. The function **both** sets the principal key and rotates internal keys as needed.
You can configure a default principal key using a global key provider. This key is used by all databases that do not have their own encryption keys configured.
There are two main functions for this:
- [pg_tde_create_key_using_global_key_provider()](../functions.md#pg_tde_create_key_using_global_key_provider) creates a principal key at a global key provider
- [pg_tde_set_default_key_using_global_key_provider()](../functions.md#pg_tde_set_default_key_using_global_key_provider) sets the default principal key and rotates the internal encryption key if one is already configured
## Create a default principal key
!!! note
The sample output below is for demonstration purposes only. Be sure to replace the key name and provider with your actual values.
To create a global principal key, run:
@ -57,6 +63,7 @@ SELECT pg_tde_set_default_key_using_global_key_provider(
* `global_vault_provider` is the name of the global key provider you previously configured.
!!! note
If no error is reported, the action completed successfully.
## How key generation works
@ -64,8 +71,11 @@ SELECT pg_tde_set_default_key_using_global_key_provider(
The key material (actual cryptographic key) is auto-generated by `pg_tde` and stored securely by the configured provider.
!!! note
This process sets the **default principal key for the entire server**. Any database without a key explicitly configured will fall back to this key.
## Next steps
To confirm that encryption is working as expected, follow the validation steps:
[Validate Encryption with pg_tde :material-arrow-right:](../test.md){.md-button}

@ -1,6 +1,6 @@
# Vault configuration
You can configure `pg_tde` to use HashiCorp Vault as a global key provider for managing encryption keys securely.
You can configure `pg_tde` to use HashiCorp Vault as a global key provider for managing encryption keys securely. Both the open source and enterprise editions are supported.
!!! note
This guide assumes that your Vault server is already set up and accessible. Vault configuration is outside the scope of this document, see [Vault's official documentation](https://developer.hashicorp.com/vault/docs) for more information.

@ -0,0 +1,77 @@
# Backup with WAL encryption enabled
To create a backup with WAL encryption enabled:
1. Copy the `pg_tde` directory from the source server’s data directory, for example `/var/lib/postgresql/data/pg_tde/`, including the `wal_keys` and `1664_providers` files, to the backup destination directory where `pg_basebackup` will write the backup.
Also copy any external files referenced by your providers configuration (such as certificate or key files) into the same relative paths under the backup destination, so that they are located and validated by `pg_basebackup -E`.
2. Run:
```bash
pg_basebackup -D /path/to/backup -E
```
Where:
- `-D /path/to/backup` specifies the backup location where you have to copy `pg_tde`.
- `-E` (or `--encrypt-wal`) enables WAL encryption and validates that the copied `pg_tde` and provider files are present and that the server key is accessible (required).
!!! note
- The `-E` flag only works with the `-X stream` option (default). It is not compatible with `-X none` or `-X fetch`. For more information, see [the other WAL methods topic](#other-wal-methods).
- The `-E` flag is only supported with the plain output format (`-F p`). It cannot be used with the tar output format (`-F t`).
## Key rotation during backups
!!! warning
Do not create, change, or rotate global key providers (or their keys) while `pg_basebackup` is running. Standbys or standalone clusters created from such backups may fail to start during WAL replay and may also lead to the corruption of encrypted data (tables, indexes, and other relations).
Creating, changing, or rotating global key providers (or their keys) during a base backup can leave the standby in an inconsistent state where it cannot retrieve the correct key history.
For example, you may see errors such as:
```sql
FATAL: failed to retrieve principal key "database_keyXXXX" from key provider "providerYYYY"
CONTEXT: WAL redo at ... ROTATE_PRINCIPAL_KEY ...
```
To ensure standby recoverability, plan key rotations outside backup windows or take a new full backup after rotation completes.
## Restore a backup created with WAL encryption
When you want to restore a backup created with `pg_basebackup -E`:
1. Ensure all external files referenced by your providers configuration (such as certificates or key files) are also present and accessible at the same relative paths.
2. Start PostgreSQL with the restored data directory.
## Backup method compatibility with WAL encryption
Tar format (`-F t`):
* Works with `-X fetch`.
* Does not support `-X stream` when WAL encryption is enabled. Using `pg_basebackup -F t -X stream` will create a broken replica.
Streaming mode (`-X stream`):
* **Must** specify `-E` (`--encrypt-wal`).
* Without `-E`, backups may contain decrypted WAL while `wal_encryption=on` remains in `postgresql.conf` and `pg_tde/wal_keys` are copied. This leads to **startup failures and compromised data in the backup**.
Fetch mode (`-X fetch`):
* Compatible with encrypted WAL without requiring any additional flags.
None (`-X none`):
* Excludes WAL from the backup and is unaffected by WAL encryption.
!!! note
If the source server has `pg_tde/wal_keys`, running `pg_basebackup` with `-X none` or `-X fetch` produces warnings such as:
```sql
pg_basebackup: warning: the source has WAL keys, but no WAL encryption configured for the target backups
pg_basebackup: detail: This may lead to exposed data and broken backup
pg_basebackup: hint: Run pg_basebackup with -E to encrypt streamed WAL
```
You can safely ignore the warnings with `-X none` or `-X fetch`, since no WAL streaming occurs.

@ -6,15 +6,9 @@ To encrypt the data, two types of keys are used:
* The **principal key** to encrypt database keys. It is kept separately from the database keys and is managed externally in the key management store.
!!! note
For more information on managing and storing principal keys externally, including supported key management systems and the local keyring option, see [Key management overview](../global-key-provider-configuration/overview.md).
For more information on managing and storing principal keys externally, see [Configure Global Key Provider](../global-key-provider-configuration/overview.md).
You have the following options to store and manage principal keys externally:
* Use the HashiCorp Vault server. Only the back end KV Secrets Engine - Version 2 (API) is supported.
* Use the KMIP-compatible server. `pg_tde` has been tested with the [PyKMIP](https://pykmip.readthedocs.io/en/latest/server.html) server and [the HashiCorp Vault Enterprise KMIP Secrets Engine](https://www.vaultproject.io/docs/secrets/kmip).
The encryption process is the following:
The encryption process works as follows:
![image](../_images/tde-flow.png)

@ -1,8 +1,109 @@
# Limitations of pg_tde
The following are current limitations of `pg_tde`:
Limitations of `pg_tde` {{release}}:
* System tables, which include statistics data and database statistics, are currently **not encrypted**.
* The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**.
* PostgreSQL’s internal system tables, which include statistics and metadata, are not encrypted.
* Temporary files created when queries exceed `work_mem` are not encrypted. These files may persist during long-running queries or after a server crash which can expose sensitive data in plaintext on disk.
## Currently unsupported WAL tools
The following tools are currently unsupported with `pg_tde` WAL encryption:
* `pg_createsubscriber`
* `pg_receivewal`
* `pg_verifybackup` by default fails with checksum or WAL key size mismatch errors.
As a workaround, use `-s` (skip checksum) and `-n` (`--no-parse-wal`) to verify backups.
The following tools and extensions in Percona Distribution for PostgreSQL have been tested and verified to work with `pg_tde` WAL encryption:
## Supported WAL tools
The following tools have been tested and verified by Percona to work with `pg_tde` WAL encryption:
* Patroni, for an example configuration see the following [Patroni configuration file](#example-patroni-configuration)
* `pg_basebackup` (with `--wal-method=stream` or `--wal-method=none`), for details on using `pg_basebackup` with WAL encryption, see [Backup with WAL encryption enabled](../how-to/backup-wal-enabled.md)
* `pg_resetwal`
* `pg_rewind`
* `pg_upgrade`
* `pg_waldump`
* pgBackRest
## Example Patroni configuration
The following is a Percona-tested example configuration.
??? example "Click to expand the Percona-tested Patroni configuration"
```yaml
# Example Patroni configuration file maintained by Percona
# Source: https://github.com/jobinau/pgscripts/blob/main/patroni/patroni.yml
scope: tde
name: pg1
restapi:
listen: 0.0.0.0:8008
connect_address: pg1:8008
etcd3:
host: etcd1:2379
bootstrap:
dcs:
ttl: 30
loop_wait: 10
retry_timeout: 10
maximum_lag_on_failover: 1048576
postgresql:
use_pg_rewind: true
use_slots: true
parameters:
archive_command: "/lib/postgresql/17/bin/pg_tde_archive_decrypt %f %p \"pgbackrest --stanza=tde archive-push %%p\""
archive_timeout: 600s
archive_mode: "on"
logging_collector: "on"
restore_command: "/lib/postgresql/17/bin/pg_tde_restore_encrypt %f %p \"pgbackrest --stanza=tde archive-get %%f \\\"%%p\\\"\""
pg_hba:
- local all all peer
- host all all 0.0.0.0/0 scram-sha-256
- host all all ::/0 scram-sha-256
- local replication all peer
- host replication all 0.0.0.0/0 scram-sha-256
- host replication all ::/0 scram-sha-256
initdb:
- encoding: UTF8
- data-checksums
- set: shared_preload_libraries=pg_tde
post_init: /usr/local/bin/setup_cluster.sh
postgresql:
listen: 0.0.0.0:5432
connect_address: pg1:5432
data_dir: /var/lib/postgresql/patroni-17
bin_dir: /lib/postgresql/17/bin
pgpass: /var/lib/postgresql/patronipass
authentication:
replication:
username: replicator
password: rep-pass
superuser:
username: postgres
password: secretpassword
parameters:
unix_socket_directories: /tmp
# Use unix_socket_directories: /var/run/postgresql for Debian/Ubuntu distributions
watchdog:
mode: off
tags:
nofailover: false
noloadbalance: false
clonefrom: false
nosync: false
```
!!! warning
The above example is Percona-tested, but Patroni versions differ, especially with discovery backends such as `etcd`. Ensure you adjust the configuration to match your environment, version, and security requirements.
## Next steps
Check which PostgreSQL versions and deployment types are compatible with `pg_tde` before planning your installation.
[View the versions and supported deployments :material-arrow-right:](supported-versions.md){.md-button}
Begin the installation process when you're ready to set up encryption.
[Start installing `pg_tde`](../install.md){.md-button}

@ -1,6 +1,6 @@
# pg_tde 1.0 ({{date.GA10}})
The `pg_tde` by Percona extension brings in [Transparent Data Encryption (TDE) :octicons-link-external-16:](../index/index.md) to PostgreSQL and enables you to keep sensitive data safe and secure.
The `pg_tde` by Percona extension brings in [Transparent Data Encryption (TDE) :octicons-link-external-16:](../index/about-tde.md) to PostgreSQL and enables you to keep sensitive data safe and secure.
[Get Started](../install.md){.md-button}
@ -8,7 +8,7 @@ The `pg_tde` by Percona extension brings in [Transparent Data Encryption (TDE) :
* **`pg_tde` 1.0 is now GA (Generally Available)**
And **stable** for encrypting relational data in PostgreSQL using [Transparent Data Encryption (TDE) :octicons-link-external-16:](../index/index.md). This milestone brings production-level data protection to PostgreSQL workloads.
And **stable** for encrypting relational data in PostgreSQL using [Transparent Data Encryption (TDE) :octicons-link-external-16:](../index/about-tde.md). This milestone brings production-level data protection to PostgreSQL workloads.
* **WAL encryption is still in Beta**
@ -16,7 +16,7 @@ The WAL encryption feature is currently still in beta and is not effective unles
## Upgrade considerations
`pg_tde` ({{tdeversion}}) is **not** backward compatible with previous `pg_tde` versions, like Release Candidate 2, due to significant changes in code. This means you **cannot** directly upgrade from one version to another. You must do **a clean installation** of `pg_tde`.
`pg_tde` 1.0 is **not** backward compatible with previous `pg_tde` versions, like Release Candidate 2, due to significant changes in code. This means you **cannot** directly upgrade from one version to another. You must do **a clean installation** of `pg_tde`.
## Known issues

@ -0,0 +1,82 @@
# pg_tde 2.0 ({{date.GA20}})
The `pg_tde` by Percona extension brings [Transparent Data Encryption (TDE)](../index/about-tde.md) to PostgreSQL and enables you to keep sensitive data safe and secure.
[Get Started](../install.md){.md-button}
## Release Highlights
### WAL encryption is now generally available
The WAL (Write-Ahead Logging) encryption feature is now fully supported and production-ready, it adds secure write-ahead logging to `pg_tde`, expanding Percona's PostgreSQL encryption coverage by enabling secure, transparent encryption of write-ahead logs using the same key infrastructure as data encryption.
### WAL encryption upgrade limitation
Clusters that used WAL encryption in the beta release (`pg_tde` 1.0 or older) cannot be upgraded to `pg_tde` 2.0. The following error indicates that WAL encryption was enabled:
```sql
FATAL: principal key not configured
HINT: Use pg_tde_set_server_key_using_global_key_provider() to configure one.
```
Clusters that did not use WAL encryption in beta can be upgraded normally.
### Documentation updates
* Updated the [Limitations](../index/tde-limitations.md) topic, it now includes WAL encryption limitations and both supported and unsupported WAL tools
* [PG-1858 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1858) - Added a new topic for [Backup with WAL encryption enabled](../how-to/backup-wal-enabled.md) that includes restoring a backup created with WAL encryption
* [PG-1832 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1858) - Added documentation for using the `pg_tde_archive_decrypt` and `pg_tde_restore_encrypt` utilities. These tools are now covered in [CLI Tools](../command-line-tools/cli-tools.md) to guide users on how to archive and restore encrypted WAL segments securely
* [PG-1740 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1740) - Updated documentation for [uninstalling `pg_tde`](../how-to/uninstall.md) with WAL encryption enabled and improved the uninstall instructions to cover cases where TDE is disabled while WAL encryption remains active
## Known issues
* Creating, changing, or rotating global key providers (or their keys) while `pg_basebackup` is running may cause standbys or standalone clusters initialized from the backup to fail during WAL replay and may also lead to the corruption of encrypted data (tables, indexes, and other relations).
Avoid making these actions during backup windows. Run a new full backup after completing a rotation or provider update.
* Using `pg_basebackup` with `--wal-method=fetch` produces warnings.
This behavior is expected and will be addressed in a future release.
* The default `mlock` limit on Rocky Linux 8 for ARM64-based architectures equals the memory page size and is 64 Kb. This results in the child process with `pg_tde` failing to allocate another memory page because the max memory limit is reached by the parent process.
To prevent this, you can change the `mlock` limit to be at least twice bigger than the memory page size:
* temporarily for the current session using the `ulimit -l <value>` command.
* set a new hard limit in the `/etc/security/limits.conf` file. To do so, you require the superuser privileges.
Adjust the limits with caution since it affects other processes running in your system.
## Changelog
### New Features
* [PG-1497 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1497) WAL encryption is now generally available (GA)
* [PG-1037 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1037) - Added support for `pg_rewind` with encrypted WAL
* [PG-1411 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1497) - Added support for `pg_resetwal` with encrypted WAL
* [PG-1603 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1603) - Added support for `pg_basebackup` with encrypted WAL
* [PG-1710 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1710) - Added support for WAL archiving with encrypted WAL
* [PG-1711 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1711) - Added support for incremental backups with encrypted WAL, compatibility has been verified with `pg_combinebackup` and the WAL summarizer tool.
* [PG-1712 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1712) - Added support for `pg_createsubscriber` with encrypted WAL
* [PG-1833 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1833) - Added verified support for using `pg_waldump` with encrypted WAL
* [PG-1834 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1834) - Verified `pg_upgrade` with encryption
### Improvements
* [PG-1661 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1661) - Added validation for key material received from providers
* [PG-1667 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1667) - Validated Vault keyring engine type
### Bugs Fixed
* [PG-1391 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1391) - Fixed unencrypted checkpoint segment on replica with encrypted key
* [PG-1412 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1412) – Fixed an issue where `XLogFileCopy` failed with encrypted WAL during PITR and `pg_rewind`
* [PG-1452 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1452) - Fixed an issue where `pg_tde_change_key_provider` did not work without the `-D` flag even if `PGDATA` was set
* [PG-1485 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1485) - Fixed an issue where streaming replication failed with an invalid magic number in WAL when `wal_encryption` was enabled
* [PG-1604 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1604) - Fixed a crash during standby promotion caused by an invalid magic number when replaying two-phase transactions from WAL
* [PG-1658 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1658) - Fixed an issue where the global key provider could not be deleted after server restart
* [PG-1835 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1835) - Fixed an issue where `pg_resetwal` corrupted encrypted WAL, causing PostgreSQL to fail at startup with an invalid checkpoint
* [PG-1842 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1842) - Fixed a delay in replica startup with encrypted tables in streaming replication setups
* [PG-1843 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1843) - Fixed performance issues when creating encrypted tables
* [PG-1863 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1863) - Fixed an issue where unnecessary WAL was generated when creating temporary tables
* [PG-1866 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1866) - Fixed an issue where automatic restart after crash sometimes failed with WAL encryption enabled
* [PG-1867 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1867) - Fixed archive recovery with encrypted WAL

@ -4,6 +4,7 @@ This page lists all release notes for `pg_tde`, organized by year and version. U
## 2025
* [2.0](release-notes-v2.0.md) ({{date.GA20}})
* [1.0](release-notes-v1.0.md) ({{date.GA10}})
* [Release Candidate 2 (RC2)](rc2.md) ({{date.RC2}})
* [Release Candidate 1 (RC1)](rc.md) ({{date.RC}})

@ -7,5 +7,5 @@
{% if config.site_description %}
<h1>{{ config.site_description }}</h1>
{% endif %}
<h2> 1.0 (2025-06-30)</h2>
<h2> 2.0 (2025-09-01)</h2>
<!--<h3>Percona Technical Documentation Team</h3>-->

@ -59,4 +59,4 @@ ALTER TABLE table_name SET ACCESS METHOD tde_heap;
## Next steps
[Configure WAL Encryption (tech preview) :material-arrow-right:](wal-encryption.md){.md-button}
[Configure WAL encryption :material-arrow-right:](wal-encryption.md){.md-button}

@ -1,7 +1,4 @@
# Configure WAL Encryption (tech preview)
!!! warning
The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**.
# Configure WAL encryption
Before enabling WAL encryption, follow the steps below to create a principal key and configure it for WAL:
@ -118,3 +115,5 @@ Now WAL files start to be encrypted for both encrypted and unencrypted tables.
For more technical references related to architecture, variables or functions, see:
[Technical Reference](advanced-topics/tech-reference.md){.md-button}
💬 Need help customizing this for your infrastructure? [Contact Percona support :octicons-link-external-16:](get-help.md)

@ -185,7 +185,7 @@ nav:
- "Keyring file configuration": global-key-provider-configuration/keyring.md
- "2.2 Global Principal Key configuration": global-key-provider-configuration/set-principal-key.md
- "3. Validate encryption with pg_tde": test.md
- "4. Configure WAL encryption (tech preview)": wal-encryption.md
- "4. Configure WAL encryption": wal-encryption.md
- "Technical reference":
- "Overview": advanced-topics/tech-reference.md
- "Architecture": architecture/architecture.md
@ -193,23 +193,26 @@ nav:
- "Functions": functions.md
- "Streaming Replication with tde_heap": replication.md
- "TDE operations":
- "pg_tde CLI Tools":
- "pg_tde CLI tools":
- "Overview": command-line-tools/cli-tools.md
- "pg_tde_change_key_provider": command-line-tools/pg-tde-change-key-provider.md
- "pg_waldump": command-line-tools/pg-waldump.md
- "pg_tde_archive_decrypt": command-line-tools/pg-tde-archive-decrypt.md
- "pg_tde_restore_encrypt": command-line-tools/pg-tde-restore-encrypt.md
- "pg_checksums": command-line-tools/pg-tde-checksums.md
- "pg_waldump": command-line-tools/pg-waldump.md
- "Uninstall pg_tde": how-to/uninstall.md
- "Configure Multi-tenancy": how-to/multi-tenant-setup.md
- "Encryption Enforcement": how-to/enforcement.md
- "Decrypt an Encrypted Table": how-to/decrypt.md
- "Backup with WAL encryption enabled": how-to/backup-wal-enabled.md
- "Restore an encrypted pg_tde backup": how-to/restore-backups.md
- faq.md
- "Release notes":
- "Release notes index": release-notes/release-notes.md
- "2025":
- "1.0": release-notes/release-notes-v1.0.md
- "Release Candidate 2": release-notes/rc2.md
- "Release Candidate 1": release-notes/rc.md
- "2.0": release-notes/release-notes-v2.0.md
- "1.0": release-notes/release-notes-v1.0.md
- "Release Candidate 2": release-notes/rc2.md
- "Release Candidate 1": release-notes/rc.md
- "2024 (Alpha 1 - Beta 2)":
- "Beta 2": release-notes/beta2.md
- "Beta": release-notes/beta.md

@ -1,12 +1,13 @@
#Variables used throughout the docs
latestreleasenotes: 'release-notes-v1.0'
tdeversion: '1.0'
release: '1.0'
latestreleasenotes: 'release-notes-v2.0'
tdeversion: '2.0'
release: '2.0'
pgversion17: '17.5'
tdebranch: release-17.5.2
tdebranch: release-17.5.3
date:
GA20: '2025-09-01'
GA10: '2025-06-30'
RC2: '2025-05-29'
RC: '2025-03-27'

@ -2,7 +2,7 @@ CREATE EXTENSION pg_tde;
SELECT pg_tde_version();
pg_tde_version
----------------
pg_tde 1.0.0
pg_tde 2.0.0
(1 row)
DROP EXTENSION pg_tde;

@ -78,6 +78,7 @@ endif
install_data(
'pg_tde.control',
'pg_tde--1.0.sql',
'pg_tde--1.0--2.0.sql',
kwargs: contrib_data_args,
)

@ -0,0 +1,4 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION pg_tde UPDATE TO '2.0'" to load this file. \quit
-- This file is a dummy, because no SQL changed between 1.0 and 2.0

@ -1,4 +1,4 @@
comment = 'pg_tde access method'
default_version = '1.0'
default_version = '2.0'
module_pathname = '$libdir/pg_tde'
relocatable = false

@ -295,10 +295,9 @@ pg_tde_perform_rotate_key(const TDEPrincipalKey *principal_key, const TDEPrincip
CloseTransientFile(new_fd);
/*
* Do the final steps - replace the current _map with the file with new
* Do the final step - replace the current _keys with the file with new
* data
*/
durable_unlink(old_path, ERROR);
durable_rename(new_path, old_path, ERROR);
/*
@ -316,8 +315,8 @@ pg_tde_perform_rotate_key(const TDEPrincipalKey *principal_key, const TDEPrincip
{
XLogPrincipalKeyRotate xlrec;
xlrec.databaseId = principal_key->keyInfo.databaseId;
xlrec.keyringId = principal_key->keyInfo.keyringId;
xlrec.databaseId = new_principal_key->keyInfo.databaseId;
xlrec.keyringId = new_principal_key->keyInfo.keyringId;
memcpy(xlrec.keyName, new_principal_key->keyInfo.name, sizeof(new_principal_key->keyInfo.name));
XLogBeginInsert();

@ -735,10 +735,9 @@ pg_tde_perform_rotate_server_key(const TDEPrincipalKey *principal_key,
CloseTransientFile(new_fd);
/*
* Do the final steps - replace the current WAL key file with the file
* with new data.
* Do the final step - replace the current WAL key file with the file with
* new data.
*/
durable_unlink(get_wal_key_file_path(), ERROR);
durable_rename(tmp_path, get_wal_key_file_path(), ERROR);
/*
@ -756,8 +755,8 @@ pg_tde_perform_rotate_server_key(const TDEPrincipalKey *principal_key,
{
XLogPrincipalKeyRotate xlrec;
xlrec.databaseId = principal_key->keyInfo.databaseId;
xlrec.keyringId = principal_key->keyInfo.keyringId;
xlrec.databaseId = new_principal_key->keyInfo.databaseId;
xlrec.keyringId = new_principal_key->keyInfo.keyringId;
memcpy(xlrec.keyName, new_principal_key->keyInfo.name, sizeof(new_principal_key->keyInfo.name));
XLogBeginInsert();

@ -2,7 +2,7 @@
#define PG_TDE_H
#define PG_TDE_NAME "pg_tde"
#define PG_TDE_VERSION "1.0.0"
#define PG_TDE_VERSION "2.0.0"
#define PG_TDE_VERSION_STRING PG_TDE_NAME " " PG_TDE_VERSION
#define PG_TDE_DATA_DIR "pg_tde"

@ -20,7 +20,7 @@ CREATE EXTENSION pg_tde;
SELECT extname, extversion FROM pg_extension WHERE extname = 'pg_tde';
extname | extversion
---------+------------
pg_tde | 1.0
pg_tde | 2.0
(1 row)
CREATE TABLE test_enc (id SERIAL, k INTEGER, PRIMARY KEY (id)) USING tde_heap;

@ -134,7 +134,7 @@ endif
pg_version_major = pg_version_arr[0].to_int()
pg_version_minor = pg_version_arr[1].to_int()
pg_version_num = (pg_version_major * 10000) + pg_version_minor
pg_percona_ver = '2'
pg_percona_ver = '3'
pg_url = 'https://www.postgresql.org/'

@ -17,7 +17,7 @@ my $pgdata = $node->data_dir;
# To make this testcase work, PERCONA_SERVER_VERSION variable should be available in environment.
# If you are using ci_scripts it is already declated in ci_scripts/env.sh
# If you are using command line make for regression then export like:
# export PERCONA_SERVER_VERSION=17.5.2
# export PERCONA_SERVER_VERSION=17.5.3
if (!defined($ENV{PERCONA_SERVER_VERSION}))
{

Loading…
Cancel
Save