From 2f87f1ea6032bb0f3a4f8f83da2b7fdbeb79b2af Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Tue, 4 Mar 2025 15:04:15 +0000 Subject: [PATCH 01/31] Remove debug leftover --- contrib/pg_tde/src/catalog/tde_keyring.c | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/pg_tde/src/catalog/tde_keyring.c b/contrib/pg_tde/src/catalog/tde_keyring.c index 793a65f000c..5e415fbf533 100644 --- a/contrib/pg_tde/src/catalog/tde_keyring.c +++ b/contrib/pg_tde/src/catalog/tde_keyring.c @@ -631,7 +631,6 @@ scan_key_provider_file(ProviderScanType scanType, void *scanKey, Oid dbOid) fd = BasicOpenFile(kp_info_path, PG_BINARY); if (fd < 0) { - fprintf(stderr, "WTF\n"); LWLockRelease(tde_provider_info_lock()); ereport(DEBUG2, (errcode_for_file_access(), From cf59a9f59c465a38c56ef0987403c9b5b8e04f14 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Tue, 4 Mar 2025 15:07:00 +0000 Subject: [PATCH 02/31] Fixing newlines in pg_tde_alter_key_provider --- .../pg_tde_alter_key_provider.c | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/bin/pg_tde_alter_key_provider/pg_tde_alter_key_provider.c b/src/bin/pg_tde_alter_key_provider/pg_tde_alter_key_provider.c index b966c9d6cbc..3802ad996d7 100644 --- a/src/bin/pg_tde_alter_key_provider/pg_tde_alter_key_provider.c +++ b/src/bin/pg_tde_alter_key_provider/pg_tde_alter_key_provider.c @@ -16,14 +16,14 @@ static void help(void) { - printf("pg_tde_modify_key_provider changes the configuration of a pg_tde key provider"); + printf("pg_tde_modify_key_provider changes the configuration of a pg_tde key provider\n"); puts(""); - printf("Usage:"); - printf("pg_tde_modify_key_provider [-D ] file "); - printf("pg_tde_modify_key_provider [-D ] vault []"); - printf("pg_tde_modify_key_provider [-D ] kmip []"); - printf("\nWARNING:"); - printf("This tool only changes the values, without properly XLogging the changes. Only use it in case the database is inaccessible and can't be started."); + printf("Usage:\n"); + printf("pg_tde_modify_key_provider [-D ] file \n"); + printf("pg_tde_modify_key_provider [-D ] vault []\n"); + printf("pg_tde_modify_key_provider [-D ] kmip []\n"); + printf("\nWARNING:\n"); + printf("This tool only changes the values, without properly XLogging the changes. Only use it in case the database is inaccessible and can't be started.\n"); } #define BUFFER_SIZE 1024 @@ -73,7 +73,7 @@ build_json(char *buffer, int count,...) } if (ptr - buffer > BUFFER_SIZE) { - printf("Error: Configuration too long."); + printf("Error: Configuration too long.\n"); return false; } } @@ -83,7 +83,7 @@ build_json(char *buffer, int count,...) if (ptr - buffer > BUFFER_SIZE) { - printf("Error: Configuration too long."); + printf("Error: Configuration too long.\n"); return false; } @@ -140,7 +140,7 @@ main(int argc, char *argv[]) { help(); puts("\n"); - printf("Error: Data directory missing"); + printf("Error: Data directory missing.\n"); exit(1); } @@ -162,7 +162,7 @@ main(int argc, char *argv[]) { help(); puts("\n"); - printf("Error: wrong number of arguments"); + printf("Error: wrong number of arguments.\n"); exit(1); } @@ -180,7 +180,7 @@ main(int argc, char *argv[]) { help(); puts("\n"); - printf("Error: wrong number of arguments"); + printf("Error: wrong number of arguments.\n"); exit(1); } @@ -198,7 +198,7 @@ main(int argc, char *argv[]) { help(); puts("\n"); - printf("Error: wrong number of arguments"); + printf("Error: wrong number of arguments.\n"); exit(1); } @@ -212,7 +212,7 @@ main(int argc, char *argv[]) { help(); puts("\n"); - printf("Error: Unknown provider type: %s", new_provider_type); + printf("Error: Unknown provider type: %s\n.", new_provider_type); exit(1); } @@ -226,7 +226,7 @@ main(int argc, char *argv[]) if (keyring == NULL) { - printf("Error: provider not found"); + printf("Error: provider not found\n."); exit(1); } @@ -235,7 +235,7 @@ main(int argc, char *argv[]) provider.provider_type = get_keyring_provider_from_typename(new_provider_type); modify_key_provider_info(&provider, db_oid, false); - printf("Key provider updated successfully!"); + printf("Key provider updated successfully!\n"); return 0; } From 3824063e66e08ee030d4a4116cd2c90886ee0cf0 Mon Sep 17 00:00:00 2001 From: Anastasia Alexandrova Date: Wed, 5 Mar 2025 12:37:35 +0100 Subject: [PATCH 03/31] Removed outdated install instructions (#101) Removed outdated install instructions: * Deleted build from source steps, * Updated Docker tab with the link to distro tutorial, * Added manual download tab with the link to Distro tutorial * Fixed broken links modified: contrib/pg_tde/documentation/docs/apt.md modified: contrib/pg_tde/documentation/docs/contribute.md modified: contrib/pg_tde/documentation/docs/faq.md modified: contrib/pg_tde/documentation/docs/install.md modified: contrib/pg_tde/documentation/docs/multi-tenant-setup.md modified: contrib/pg_tde/documentation/docs/release-notes/release-notes.md deleted: contrib/pg_tde/documentation/docs/switch.md modified: contrib/pg_tde/documentation/docs/yum.md --- contrib/pg_tde/documentation/docs/apt.md | 34 +++++----- .../pg_tde/documentation/docs/contribute.md | 2 +- contrib/pg_tde/documentation/docs/faq.md | 4 +- contrib/pg_tde/documentation/docs/install.md | 68 +++---------------- .../documentation/docs/multi-tenant-setup.md | 2 +- .../docs/release-notes/release-notes.md | 2 +- contrib/pg_tde/documentation/docs/switch.md | 5 -- contrib/pg_tde/documentation/docs/yum.md | 23 +++---- 8 files changed, 44 insertions(+), 96 deletions(-) delete mode 100644 contrib/pg_tde/documentation/docs/switch.md diff --git a/contrib/pg_tde/documentation/docs/apt.md b/contrib/pg_tde/documentation/docs/apt.md index 8a94750d972..2ecd9eb232f 100644 --- a/contrib/pg_tde/documentation/docs/apt.md +++ b/contrib/pg_tde/documentation/docs/apt.md @@ -10,7 +10,7 @@ Check the [list of supported platforms](install.md#__tabbed_1_1). 2. You need the `percona-release` repository management tool that enables the desired Percona repository for you. -### Install `percona-release` +### Install `percona-release` {.power-number} 1. You need the following dependencies to install `percona-release`: @@ -21,26 +21,26 @@ Check the [list of supported platforms](install.md#__tabbed_1_1). Install them with the following command: - ```bash - sudo apt-get install -y wget gnupg2 curl lsb-release + ```{.bash data-prompt="$"} + $ sudo apt-get install -y wget gnupg2 curl lsb-release ``` 2. Fetch the `percona-release` package - ```bash - sudo wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb + ```{.bash data-prompt="$"} + $ sudo wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb ``` 3. Install `percona-release` - ```bash - sudo dpkg -i percona-release_latest.generic_all.deb + ```{.bash data-prompt="$"} + $ sudo dpkg -i percona-release_latest.generic_all.deb ``` 4. Enable the Percona Distribution for PostgreSQL repository ```{.bash data-prompt="$"} - $ sudo percona-release enable ppg-{{pgversion17}} + $ sudo percona-release enable-only ppg-{{pgversion17}} ``` 6. Update the local cache @@ -49,30 +49,30 @@ Check the [list of supported platforms](install.md#__tabbed_1_1). sudo apt-get update ``` -## Install `pg_tde` +## Install `pg_tde` {.power-number} !!! important - The `pg_tde` {{release}} extension is a part of the `percona-postgresql-17` package. If you installed a previous version of `pg_tde` from the `percona-postgresql-17-pg-tde` package, do the following: + The `pg_tde` extension is a part of the `percona-postgresql-17` package. If you installed a previous version of `pg_tde` from the `percona-postgresql-17-pg-tde` package, do the following: 1. Drop the extension using the `DROP EXTENSION` with `CASCADE` command. - The use of the `CASCADE` parameter deletes all tables that were created in the database with `pg_tde` enabled and also all dependencies upon the encrypted table (e.g. foreign keys in a non-encrypted table used in the encrypted one). + The use of the `CASCADE` parameter deletes all tables that were created in the database with `pg_tde` enabled and also all dependencies upon the encrypted table (e.g. foreign keys in a non-encrypted table used in the encrypted one). - ```sql - DROP EXTENSION pg_tde CASCADE - ``` + ```sql + DROP EXTENSION pg_tde CASCADE + ``` 2. Uninstall the `percona-postgresql-17-pg-tde` package. After all [preconditions](#preconditions) are met, run the following command to install `pg_tde`: -```bash -sudo apt-get install -y percona-postgresql-17 +```{.bash data-prompt="$"} +$ sudo apt-get install -y percona-postgresql-17 ``` ## Next step -[Setup](setup.md){.md-button} +[Setup :material-arrow-right:](setup.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/contribute.md b/contrib/pg_tde/documentation/docs/contribute.md index ecc54fbea80..c728f333f49 100644 --- a/contrib/pg_tde/documentation/docs/contribute.md +++ b/contrib/pg_tde/documentation/docs/contribute.md @@ -64,7 +64,7 @@ You can run tests on your local machine with whatever operating system you have. ## Contribute to documentation `pg_tde` documentation is written in Markdown language, so you can [write the docs for your code changes](#write-the-docs) or -[edit the existing documentation online via GitHub](#edit-documentation-online-vi-github). If you wish to have more control over the doc process, jump to how to [edit documentation locally](#edit-documentation-locally). +[edit the existing documentation online via GitHub](#edit-documentation-online-via-github). If you wish to have more control over the doc process, jump to how to [edit documentation locally](#edit-documentation-locally). Before you start, learn what [Markdown] is and how to write it. For your convenience, there's also a [Markdown cheat sheet] to help you with the syntax. diff --git a/contrib/pg_tde/documentation/docs/faq.md b/contrib/pg_tde/documentation/docs/faq.md index 835a90f4702..50dbfa72649 100644 --- a/contrib/pg_tde/documentation/docs/faq.md +++ b/contrib/pg_tde/documentation/docs/faq.md @@ -98,9 +98,9 @@ Whenever the WAL is being read (by the recovery process or tools), the decision It depends on your business requirements and the sensitivity of your data. Encrypting all data is a good practice but it can have a performance impact. -Consider encrypting only tables that store sensitive data. You can decide what tables to encrypt and with what key. The [Setup](setup.md) section in documentation focuses on this approach. +Consider encrypting only tables that store sensitive data. You can decide what tables to encrypt and with what key. The [Set up multi-tenancy](multi-tenant-setup.md) section in the documentation focuses on this approach. -We advise encrypting the whole database only if all your data is sensitive, like PII, or if there is no other way to comply with data safety requirements. See [How to configure global encryption](global-encryption.md). +We advise encrypting the whole database only if all your data is sensitive, like PII, or if there is no other way to comply with data safety requirements. ## What cipher mechanisms are used by `pg_tde`? diff --git a/contrib/pg_tde/documentation/docs/install.md b/contrib/pg_tde/documentation/docs/install.md index e83a93ef356..76521590d1e 100644 --- a/contrib/pg_tde/documentation/docs/install.md +++ b/contrib/pg_tde/documentation/docs/install.md @@ -1,9 +1,8 @@ # Installation -Install `pg_tde` using one of available installation methods: +Install `pg_tde` using one of the available installation methods: - -=== "Package manager" +=== ":octicons-terminal-16: Package manager" The packages are available for the following operating systems: @@ -15,67 +14,22 @@ Install `pg_tde` using one of available installation methods: - Debian 11 (Bullseye) - Debian 12 (Bookworm) - [Install on Debian or Ubuntu](apt.md){.md-button} - [Install on RHEL or derivatives](yum.md){.md-button} - -=== "Build from source" - - To build `pg_tde` from source code, do the following: - - 1. On Ubuntu/Debian: Install the following dependencies required for the build: - - ```sh - sudo apt install make gcc postgresql-server-dev-17 libcurl4-openssl-dev - ``` - - 2. [Install Percona Distribution for PostgreSQL 17 :octicons-link-external-16:](https://docs.percona.com/postgresql/17/installing.html) or [upstream PostgreSQL 17 :octicons-link-external-16:](https://www.postgresql.org/download/) - - 3. If PostgreSQL is installed in a non standard directory, set the `PG_CONFIG` environment variable to point to the `pg_config` executable. - - 4. Clone the repository: - - ``` - git clone git://github.com/percona/pg_tde - ``` - - 5. Compile and install the extension - - ``` - cd pg_tde - make USE_PGXS=1 - sudo make USE_PGXS=1 install - ``` - -=== "Run in Docker" - - !!! note - - The steps below are for the deprecated PostgreSQL Community version of `pg_tde` and apply only to PostgreSQL 16. This `pg_tde` version provides only the `tde_heap_basic` access method for data encryption. - - To enjoy full encryption features, run the `pg_tde` version for Percona Server for PostgreSQL. [Use the Percona Distribution for PostgreSQL Docker image :octicons-link-external-16:](https://docs.percona.com/postgresql/17/docker.html). - - You can find Docker images on [Docker Hub](https://hub.docker.com/r/perconalab/pg_tde). Images are built on top of [postgres:16](https://hub.docker.com/_/postgres) official image. - - To run `pg_tde` in Docker, use the following command: + [Install on Debian or Ubuntu :material-arrow-right:](apt.md){.md-button} + [Install on RHEL or derivatives :material-arrow-right:](yum.md){.md-button} - ``` - docker run --name pg-tde -e POSTGRES_PASSWORD=mysecretpassword -d perconalab/pg_tde - ``` +=== ":simple-docker: Docker" - It builds and adds the `pg_tde` extension to PostgreSQL 16. The `postgresql.conf` contains the required modifications. The `pg_tde` extension is added to `template1` so that all new databases automatically have the `pg_tde` extension loaded. + `pg_tde` is a part of the Percona Distribution for PostgreSQL Docker image. Use this image to enjoy full encryption capabilities. Check below to get access to a detailed step-by-step guide. - Keys are not created automatically. You must configure a key provider and a principal key for each database where you wish to use encrypted tables. + [Run in Docker :material-arrow-right:](https://docs.percona.com/postgresql/latest/docker.html) - Connect to the container and establish the `psql` session there. Then, see the instructions in the [Setup](setup.md) section, starting with the 4th point, as the first 3 steps are already completed in the Docker image. +=== ":octicons-download-16: Manual download" - See [Docker Docs](https://hub.docker.com/_/postgres) on usage. + `pg_tde` is included in the Percona Distribution for PostgreSQL tarball. Check below to get access to a detailed step-by-step guide. - You can also build a Docker image manually with: + [Install from tarballs](https://docs.percona.com/postgresql/17/tarball.html) guide for instructions. - ``` - docker build . -f ./docker/Dockerfile -t your-image-name - ``` ## Next steps -[Setup](setup.md){.md-button} +[Setup :material-arrow-right:](setup.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/multi-tenant-setup.md b/contrib/pg_tde/documentation/docs/multi-tenant-setup.md index 43c22d249d0..6d8b0f09300 100644 --- a/contrib/pg_tde/documentation/docs/multi-tenant-setup.md +++ b/contrib/pg_tde/documentation/docs/multi-tenant-setup.md @@ -4,7 +4,7 @@ The steps below describe how to set up multi-tenancy with `pg_tde`. Multi-tenanc If you don't need multi-tenancy, use the global key provider. See the configuration steps from the [Setup](setup.md) section. -For how to enable WAL encryption, refer to the [WAL encryption](setup.md#wal-encryption) section. +For how to enable WAL encryption, refer to the [WAL encryption](wal-encryption.md) section. --8<-- "kms-considerations.md" diff --git a/contrib/pg_tde/documentation/docs/release-notes/release-notes.md b/contrib/pg_tde/documentation/docs/release-notes/release-notes.md index 25e2ada2ff3..3dc8b6e46c1 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/release-notes.md +++ b/contrib/pg_tde/documentation/docs/release-notes/release-notes.md @@ -1,6 +1,6 @@ # pg_tde release notes -`pg_tde` extension brings in [Transparent Data Encryption (TDE)](tde.md) to PostgreSQL and enables you to keep sensitive data safe and secure. +`pg_tde` extension brings in [Transparent Data Encryption (TDE)](../tde.md) to PostgreSQL and enables you to keep sensitive data safe and secure. [Get started](../install.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/switch.md b/contrib/pg_tde/documentation/docs/switch.md deleted file mode 100644 index 5a3fa2edaf0..00000000000 --- a/contrib/pg_tde/documentation/docs/switch.md +++ /dev/null @@ -1,5 +0,0 @@ -# Switch from Percona Server for PostgreSQL to PostgreSQL Community - -Percona Server for PostgreSQL and PostgreSQL Community are binary compatible and enable you to switch from one to another. Here's how: - -1. If you used the `tde_heap` (tech preview feature) access method for encryption, either re-encrypt the data using the `tde_heap_basic` access method, or [decrypt](decrypt.md) it completely \ No newline at end of file diff --git a/contrib/pg_tde/documentation/docs/yum.md b/contrib/pg_tde/documentation/docs/yum.md index bc7a9d098da..6e9396897ce 100644 --- a/contrib/pg_tde/documentation/docs/yum.md +++ b/contrib/pg_tde/documentation/docs/yum.md @@ -4,29 +4,27 @@ This tutorial shows how to install `pg_tde` with [Percona Distribution for Postg Check the [list of supported platforms](install.md#__tabbed_1_1). -## Install `percona-release` +## Install `percona-release` {.power-number} You need the `percona-release` repository management tool that enables the desired Percona repository for you. 1. Install `percona-release`: - ```bash - sudo yum -y install https://repo.percona.com/yum/percona-release-latest.noarch.rpm + ```{.bash data-prompt="$"} + $ sudo yum -y install https://repo.percona.com/yum/percona-release-latest.noarch.rpm ``` 2. Enable the repository. - Percona provides [two repositories](repo-overview.md) for Percona Distribution for PostgreSQL. We recommend enabling the Major release repository to timely receive the latest updates. - - ```bash - sudo percona-release enable-only ppg-{{pgversion17}} + ```{.bash data-prompt="$"} + $ sudo percona-release enable-only ppg-{{pgversion17}} ``` -## Install `pg_tde` +## Install `pg_tde` {.power-number} !!! important - The `pg_tde` {{release}} extension is a part of the `percona-postgresql17` package. If you installed a previous version of `pg_tde` from the `percona-pg_tde_17` package, do the following: + The `pg_tde` extension is a part of the `percona-postgresql17` package. If you installed a previous version of `pg_tde` from the `percona-pg_tde_17` package, do the following: 1. Drop the extension using the `DROP EXTENSION` with `CASCADE` command. @@ -38,11 +36,12 @@ You need the `percona-release` repository management tool that enables the desir 2. Uninstall the `percona-pg_tde_17` package. +Run the following command to install `pg_tde`: -```bash -sudo yum -y install percona-postgresql17 +```{.bash data-prompt="$"} +$ sudo yum -y install percona-postgresql17 ``` ## Next steps -[Setup](setup.md){.md-button} +[Setup :material-arrow-right:](setup.md){.md-button} From e973d95d7c34c366cc50ae0c976c42d341213b07 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Wed, 5 Mar 2025 21:57:44 +0000 Subject: [PATCH 04/31] PG-1400: Correct pg_tde_change_key_provider naming Previusly the tool was named pg_tde_alter_key_provider, but in the help message it referred to itself as pg_tde_modify_key_provider. Instead of any of that, the name should align with the online function, which i pg_tde_change_key_provider. This commit renames the tool and addresses the help message. Also, the help message is now more detailed to make the parameters cleaner. --- src/bin/Makefile | 2 +- src/bin/meson.build | 2 +- src/bin/pg_tde_alter_key_provider/.gitignore | 1 - src/bin/pg_tde_change_key_provider/.gitignore | 1 + .../Makefile | 18 +++++------ .../meson.build | 12 +++---- .../pg_tde_change_key_provider.c} | 31 ++++++++++++------- 7 files changed, 38 insertions(+), 29 deletions(-) delete mode 100644 src/bin/pg_tde_alter_key_provider/.gitignore create mode 100644 src/bin/pg_tde_change_key_provider/.gitignore rename src/bin/{pg_tde_alter_key_provider => pg_tde_change_key_provider}/Makefile (50%) rename src/bin/{pg_tde_alter_key_provider => pg_tde_change_key_provider}/meson.build (58%) rename src/bin/{pg_tde_alter_key_provider/pg_tde_alter_key_provider.c => pg_tde_change_key_provider/pg_tde_change_key_provider.c} (78%) diff --git a/src/bin/Makefile b/src/bin/Makefile index e40285c49d2..580fbf03edc 100644 --- a/src/bin/Makefile +++ b/src/bin/Makefile @@ -31,7 +31,7 @@ SUBDIRS = \ pg_dump \ pg_resetwal \ pg_rewind \ - pg_tde_alter_key_provider \ + pg_tde_change_key_provider \ pg_test_fsync \ pg_test_timing \ pg_upgrade \ diff --git a/src/bin/meson.build b/src/bin/meson.build index 056f442d399..fe4f00d634b 100644 --- a/src/bin/meson.build +++ b/src/bin/meson.build @@ -12,7 +12,7 @@ subdir('pg_ctl') subdir('pg_dump') subdir('pg_resetwal') subdir('pg_rewind') -subdir('pg_tde_alter_key_provider') +subdir('pg_tde_change_key_provider') subdir('pg_test_fsync') subdir('pg_test_timing') subdir('pg_upgrade') diff --git a/src/bin/pg_tde_alter_key_provider/.gitignore b/src/bin/pg_tde_alter_key_provider/.gitignore deleted file mode 100644 index 5896f859363..00000000000 --- a/src/bin/pg_tde_alter_key_provider/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/pg_tde_alter_key_provider diff --git a/src/bin/pg_tde_change_key_provider/.gitignore b/src/bin/pg_tde_change_key_provider/.gitignore new file mode 100644 index 00000000000..028680dcbc3 --- /dev/null +++ b/src/bin/pg_tde_change_key_provider/.gitignore @@ -0,0 +1 @@ +/pg_tde_change_key_provider diff --git a/src/bin/pg_tde_alter_key_provider/Makefile b/src/bin/pg_tde_change_key_provider/Makefile similarity index 50% rename from src/bin/pg_tde_alter_key_provider/Makefile rename to src/bin/pg_tde_change_key_provider/Makefile index fecd253409b..86f482ae7a3 100644 --- a/src/bin/pg_tde_alter_key_provider/Makefile +++ b/src/bin/pg_tde_change_key_provider/Makefile @@ -1,13 +1,13 @@ -# src/bin/pg_tde_alter_key_provider/Makefile +# src/bin/pg_tde_change_key_provider/Makefile -PGFILEDESC = "pg_tde_alter_key_provider" +PGFILEDESC = "pg_tde_change_key_provider" -subdir = src/bin/pg_tde_alter_key_provider +subdir = src/bin/pg_tde_change_key_provider top_builddir = ../../.. include $(top_builddir)/src/Makefile.global OBJS = \ - pg_tde_alter_key_provider.o + pg_tde_change_key_provider.o override CPPFLAGS := -DFRONTEND $(CPPFLAGS) @@ -17,20 +17,20 @@ OBJS += \ override CPPFLAGS := -I$(top_srcdir)/contrib/pg_tde/src/include -I$(top_srcdir)/contrib/pg_tde/src/libkmip/libkmip/include $(CPPFLAGS) -all: pg_tde_alter_key_provider +all: pg_tde_change_key_provider -pg_tde_alter_key_provider: $(OBJS) | submake-libpgport +pg_tde_change_key_provider: $(OBJS) | submake-libpgport $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) install: all installdirs - $(INSTALL_PROGRAM) pg_tde_alter_key_provider$(X) '$(DESTDIR)$(bindir)/pg_tde_alter_key_provider$(X)' + $(INSTALL_PROGRAM) pg_tde_change_key_provider$(X) '$(DESTDIR)$(bindir)/pg_tde_change_key_provider$(X)' installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: - rm -f '$(DESTDIR)$(bindir)/pg_tde_alter_key_provider$(X)' + rm -f '$(DESTDIR)$(bindir)/pg_tde_change_key_provider$(X)' clean distclean: - rm -f pg_tde_alter_key_provider$(X) $(OBJS) $(RMGRDESCSOURCES) xlogreader.c xlogstats.c + rm -f pg_tde_change_key_provider$(X) $(OBJS) $(RMGRDESCSOURCES) xlogreader.c xlogstats.c rm -rf tmp_check \ No newline at end of file diff --git a/src/bin/pg_tde_alter_key_provider/meson.build b/src/bin/pg_tde_change_key_provider/meson.build similarity index 58% rename from src/bin/pg_tde_alter_key_provider/meson.build rename to src/bin/pg_tde_change_key_provider/meson.build index be00714b1ca..945572b4e64 100644 --- a/src/bin/pg_tde_alter_key_provider/meson.build +++ b/src/bin/pg_tde_change_key_provider/meson.build @@ -1,22 +1,22 @@ -pg_tde_alter_key_provider_sources = files( - 'pg_tde_alter_key_provider.c', +pg_tde_change_key_provider_sources = files( + 'pg_tde_change_key_provider.c', ) link_w = [pg_tde_frontend] include_dirs = [postgres_inc, pg_tde_inc] -pg_tde_alter_key_provider = executable('pg_tde_alter_key_provider', - pg_tde_alter_key_provider_sources, +pg_tde_change_key_provider = executable('pg_tde_change_key_provider', + pg_tde_change_key_provider_sources, dependencies: [frontend_code, lz4, zstd], c_args: ['-DFRONTEND'], # needed for xlogreader et al kwargs: default_bin_args, include_directories: include_dirs, link_with: link_w ) -bin_targets += pg_tde_alter_key_provider +bin_targets += pg_tde_change_key_provider tests += { - 'name': 'pg_tde_alter_key_provider', + 'name': 'pg_tde_change_key_provider', 'sd': meson.current_source_dir(), 'bd': meson.current_build_dir(), 'tap': { diff --git a/src/bin/pg_tde_alter_key_provider/pg_tde_alter_key_provider.c b/src/bin/pg_tde_change_key_provider/pg_tde_change_key_provider.c similarity index 78% rename from src/bin/pg_tde_alter_key_provider/pg_tde_alter_key_provider.c rename to src/bin/pg_tde_change_key_provider/pg_tde_change_key_provider.c index 3802ad996d7..8d270e4fb56 100644 --- a/src/bin/pg_tde_alter_key_provider/pg_tde_alter_key_provider.c +++ b/src/bin/pg_tde_change_key_provider/pg_tde_change_key_provider.c @@ -10,20 +10,29 @@ #include #include -/* version string we expect back from pg_tde_modify_key_provider */ -#define PROGNAME "pg_tde_modify_key_provider (PostgreSQL) " PG_VERSION "\n" +/* version string we expect back from pg_tde_change_key_provider */ +#define PROGNAME "pg_tde_change_key_provider (PostgreSQL) " PG_VERSION "\n" static void help(void) { - printf("pg_tde_modify_key_provider changes the configuration of a pg_tde key provider\n"); + puts("pg_tde_change_key_provider changes the configuration of a pg_tde key provider"); puts(""); - printf("Usage:\n"); - printf("pg_tde_modify_key_provider [-D ] file \n"); - printf("pg_tde_modify_key_provider [-D ] vault []\n"); - printf("pg_tde_modify_key_provider [-D ] kmip []\n"); - printf("\nWARNING:\n"); - printf("This tool only changes the values, without properly XLogging the changes. Only use it in case the database is inaccessible and can't be started.\n"); + puts("Usage:"); + puts(""); + puts("pg_tde_change_key_provider [-D ] "); + puts(""); + puts(" Where can be file, vault or kmip"); + puts(""); + puts("Depending on the provider type, the complete parameter list is:"); + puts(""); + puts("pg_tde_change_key_provider [-D ] file "); + puts("pg_tde_change_key_provider [-D ] vault []"); + puts("pg_tde_change_key_provider [-D ] kmip []"); + puts(""); + puts("WARNING:"); + puts(""); + puts("This tool only changes the values, without properly XLogging the changes, or adjusting the configuration in the running postgres processes. Only use it in case the database is inaccessible and can't be started.\n"); } #define BUFFER_SIZE 1024 @@ -110,7 +119,7 @@ main(int argc, char *argv[]) pg_logging_init(argv[0]); pg_logging_set_level(PG_LOG_WARNING); - set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_tde_alter_key_provider")); + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_tde_change_key_provider")); if (argc > 1) { @@ -121,7 +130,7 @@ main(int argc, char *argv[]) } if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) { - puts("pg_tde_alter_key_provider (PostgreSQL) " PG_VERSION); + puts("pg_tde_change_key_provider (PostgreSQL) " PG_VERSION); exit(0); } } From 0093413b362b9cdc23f37b271f4922788dc4bde4 Mon Sep 17 00:00:00 2001 From: Anastasia Alexandrova Date: Mon, 10 Mar 2025 09:18:03 +0100 Subject: [PATCH 05/31] PG-1420 Fixed function names for Inspections (#117) --- contrib/pg_tde/documentation/docs/functions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index aa0c8520d4b..436c1356629 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -36,8 +36,8 @@ These functions allow or revoke the use of the permissions management functions: Use these functions to grant or revoke the use of query functions, which do not modify the encryption settings: -* `pg_tde_grant_key_viewer_management_to_role(role)` -* `pg_tde_revoke_key_viewer_management_from_role(role)` +* `pg_tde_grant_key_viewer_to_role(role)` +* `pg_tde_revoke_key_viewer_from_role(role)` ## Key provider management From a2faa6b5908af8768a7847e7e9164376020ef344 Mon Sep 17 00:00:00 2001 From: Kai Wagner Date: Mon, 10 Mar 2025 09:01:32 +0100 Subject: [PATCH 06/31] fix typo PG-1425 Signed-off-by: Kai Wagner --- contrib/pg_tde/documentation/docs/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index 436c1356629..3358298271f 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -264,7 +264,7 @@ You can also verify if the table in a custom schema is encrypted. Pass the schem SELECT pg_tde_is_encrypted('schema.table_name'); ``` -This can additoonally be used the verify that indexes and sequences are encrypted. +This can additionally be used the verify that indexes and sequences are encrypted. ### pg_tde_principal_key_info From 629486b33b3ca08505bd1bc3c3395c052d9b0c6a Mon Sep 17 00:00:00 2001 From: Kai Wagner Date: Mon, 10 Mar 2025 09:32:15 +0100 Subject: [PATCH 07/31] grammar update Signed-off-by: Kai Wagner --- contrib/pg_tde/documentation/docs/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index 3358298271f..4dd55b12508 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -264,7 +264,7 @@ You can also verify if the table in a custom schema is encrypted. Pass the schem SELECT pg_tde_is_encrypted('schema.table_name'); ``` -This can additionally be used the verify that indexes and sequences are encrypted. +This can additionally be used to verify that indexes and sequences are encrypted. ### pg_tde_principal_key_info From 67a7f179590074ec98a3df4e670c72d628b72828 Mon Sep 17 00:00:00 2001 From: Kai Wagner Date: Mon, 10 Mar 2025 10:01:04 +0100 Subject: [PATCH 08/31] addressed rephrase from PG-1418 Signed-off-by: Kai Wagner --- contrib/pg_tde/documentation/docs/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index 4dd55b12508..4524cf8ec53 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -234,7 +234,7 @@ The `ensure_new_key` parameter instructs the function how to handle a principal Creates or rotates the default principal key for the server using the specified key provider. -The default key is automatically used as a principal key by any database that doesn't have a specific principal key created the first time an encrypted database object is created. +The default key is automatically used as a principal key by any database that doesn't have a specific principal key already created when the first encrypted database object is created. ``` SELECT pg_tde_set_default_principal_key('name-of-the-principal-key','provider-name','ensure_new_key'); From 5b6c956ec74d574e9e594aa9f6c94ec8021fad22 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Fri, 7 Mar 2025 17:24:03 +0000 Subject: [PATCH 09/31] pg_tde_change_key_provider segfault fix The tool possibly segfaulted when it received 3 incomplete arguments. With this change it properly prints the help message instead. --- src/bin/pg_tde_change_key_provider/pg_tde_change_key_provider.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/pg_tde_change_key_provider/pg_tde_change_key_provider.c b/src/bin/pg_tde_change_key_provider/pg_tde_change_key_provider.c index 8d270e4fb56..1bdb3870e97 100644 --- a/src/bin/pg_tde_change_key_provider/pg_tde_change_key_provider.c +++ b/src/bin/pg_tde_change_key_provider/pg_tde_change_key_provider.c @@ -153,7 +153,7 @@ main(int argc, char *argv[]) exit(1); } - if (argc - argstart < 3) + if (argc - argstart <= 3) { help(); exit(1); From d6dd926b32ee9729b763edbe6adf54b0e8ad119e Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Sat, 8 Mar 2025 21:57:35 +0000 Subject: [PATCH 10/31] PG-1421: Fix typo in permission management code --- contrib/pg_tde/pg_tde--1.0-rc.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pg_tde/pg_tde--1.0-rc.sql b/contrib/pg_tde/pg_tde--1.0-rc.sql index 0285eacf885..13f544372bf 100644 --- a/contrib/pg_tde/pg_tde--1.0-rc.sql +++ b/contrib/pg_tde/pg_tde--1.0-rc.sql @@ -595,7 +595,7 @@ BEGIN EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_set_global_principal_key(text, text, BOOLEAN) TO %I', target_role); EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_set_server_principal_key(text, text, BOOLEAN) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_set_default_principal_key(text, text, BOOLEAN) FROM %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_set_default_principal_key(text, text, BOOLEAN) TO %I', target_role); END; $$; From 23c93b93c0b71acddee9629363e052e305802b4d Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Sat, 8 Mar 2025 21:50:14 +0000 Subject: [PATCH 11/31] PG-1401: correct encryption status during alter table set access method Previously some files inherited the previous encryption status, for example leaving primary keys encrypted if they were ever encrypted before. This commit fixes the issue by basing new encryption status only on the new encryption setting. --- .../pg_tde/expected/change_access_method.out | 28 +++++++- .../expected/change_access_method_basic.out | 66 +++++++++++++------ contrib/pg_tde/sql/change_access_method.inc | 8 +++ .../pg_tde/src/include/pg_tde_event_capture.h | 5 +- contrib/pg_tde/src/pg_tde_event_capture.c | 14 +++- contrib/pg_tde/src/smgr/pg_tde_smgr.c | 2 +- 6 files changed, 94 insertions(+), 29 deletions(-) diff --git a/contrib/pg_tde/expected/change_access_method.out b/contrib/pg_tde/expected/change_access_method.out index 5da9c7348d7..a63611b45fb 100644 --- a/contrib/pg_tde/expected/change_access_method.out +++ b/contrib/pg_tde/expected/change_access_method.out @@ -43,6 +43,12 @@ SELECT pg_tde_is_encrypted('country_table_country_id_seq'); t (1 row) +SELECT pg_tde_is_encrypted('country_table_pkey'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + -- Try changing the encrypted table to an unencrypted table ALTER TABLE country_table SET access method heap; SELECT pg_tde_is_encrypted('country_table_country_id_seq'); @@ -51,6 +57,12 @@ SELECT pg_tde_is_encrypted('country_table_country_id_seq'); f (1 row) +SELECT pg_tde_is_encrypted('country_table_pkey'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + -- Insert some more data INSERT INTO country_table (country_name, continent) VALUES ('France', 'Europe'), @@ -79,6 +91,12 @@ SELECT pg_tde_is_encrypted('country_table_country_id_seq'); f (1 row) +SELECT pg_tde_is_encrypted('country_table_pkey'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + -- Change it back to encrypted ALTER TABLE country_table SET access method :tde_am; INSERT INTO country_table (country_name, continent) @@ -111,6 +129,12 @@ SELECT pg_tde_is_encrypted('country_table_country_id_seq'); t (1 row) +SELECT pg_tde_is_encrypted('country_table_pkey'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + ALTER TABLE country_table ADD y text; SELECT pg_tde_is_encrypted(('pg_toast.pg_toast_' || 'country_table'::regclass::oid)::regclass); pg_tde_is_encrypted @@ -129,10 +153,10 @@ CREATE TABLE country_table3 ( country_name text unique not null, continent text not null ) USING heap; -psql:sql/change_access_method.inc:67: ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. +psql:sql/change_access_method.inc:75: ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. ALTER TABLE country_table SET access method heap; -psql:sql/change_access_method.inc:69: ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. +psql:sql/change_access_method.inc:77: ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. ALTER TABLE country_table2 SET access method :tde_am; CREATE TABLE country_table3 ( country_id serial primary key, diff --git a/contrib/pg_tde/expected/change_access_method_basic.out b/contrib/pg_tde/expected/change_access_method_basic.out index 89b08c5c308..146e82396fc 100644 --- a/contrib/pg_tde/expected/change_access_method_basic.out +++ b/contrib/pg_tde/expected/change_access_method_basic.out @@ -50,15 +50,27 @@ SELECT pg_tde_is_encrypted('country_table_country_id_seq'); f (1 row) +SELECT pg_tde_is_encrypted('country_table_pkey'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + -- Try changing the encrypted table to an unencrypted table ALTER TABLE country_table SET access method heap; -psql:sql/change_access_method.inc:24: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:26: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. SELECT pg_tde_is_encrypted('country_table_country_id_seq'); pg_tde_is_encrypted --------------------- f (1 row) +SELECT pg_tde_is_encrypted('country_table_pkey'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + -- Insert some more data INSERT INTO country_table (country_name, continent) VALUES ('France', 'Europe'), @@ -87,26 +99,32 @@ SELECT pg_tde_is_encrypted('country_table_country_id_seq'); f (1 row) +SELECT pg_tde_is_encrypted('country_table_pkey'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + -- Change it back to encrypted ALTER TABLE country_table SET access method :tde_am; -psql:sql/change_access_method.inc:40: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. -psql:sql/change_access_method.inc:40: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. -psql:sql/change_access_method.inc:40: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. -psql:sql/change_access_method.inc:40: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. -psql:sql/change_access_method.inc:40: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. -psql:sql/change_access_method.inc:40: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. -psql:sql/change_access_method.inc:40: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. -psql:sql/change_access_method.inc:40: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. -psql:sql/change_access_method.inc:40: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:46: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:46: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:46: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:46: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:46: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:46: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:46: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:46: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:46: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. INSERT INTO country_table (country_name, continent) VALUES ('China', 'Asia'), ('Brazil', 'South America'), ('Australia', 'Oceania'); -psql:sql/change_access_method.inc:45: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. -psql:sql/change_access_method.inc:45: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. -psql:sql/change_access_method.inc:45: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:51: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:51: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:51: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. SELECT * FROM country_table; -psql:sql/change_access_method.inc:46: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:52: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. country_id | country_name | continent ------------+--------------+--------------- 1 | Japan | Asia @@ -132,9 +150,15 @@ SELECT pg_tde_is_encrypted('country_table_country_id_seq'); f (1 row) +SELECT pg_tde_is_encrypted('country_table_pkey'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + ALTER TABLE country_table ADD y text; -psql:sql/change_access_method.inc:51: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. -psql:sql/change_access_method.inc:51: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:59: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/change_access_method.inc:59: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. SELECT pg_tde_is_encrypted(('pg_toast.pg_toast_' || 'country_table'::regclass::oid)::regclass); pg_tde_is_encrypted --------------------- @@ -152,21 +176,21 @@ CREATE TABLE country_table3 ( country_name text unique not null, continent text not null ) USING heap; -psql:sql/change_access_method.inc:67: ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. +psql:sql/change_access_method.inc:75: ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. ALTER TABLE country_table SET access method heap; -psql:sql/change_access_method.inc:69: ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. +psql:sql/change_access_method.inc:77: ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. ALTER TABLE country_table2 SET access method :tde_am; -psql:sql/change_access_method.inc:71: ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. +psql:sql/change_access_method.inc:79: ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. CREATE TABLE country_table3 ( country_id serial primary key, country_name text unique not null, continent text not null ) using :tde_am; -psql:sql/change_access_method.inc:77: ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. +psql:sql/change_access_method.inc:85: ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. DROP TABLE country_table; DROP TABLE country_table2; DROP TABLE country_table3; -psql:sql/change_access_method.inc:81: ERROR: table "country_table3" does not exist +psql:sql/change_access_method.inc:89: ERROR: table "country_table3" does not exist SET pg_tde.enforce_encryption = OFF; DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/sql/change_access_method.inc b/contrib/pg_tde/sql/change_access_method.inc index ed486df3a13..322f1bebf8d 100644 --- a/contrib/pg_tde/sql/change_access_method.inc +++ b/contrib/pg_tde/sql/change_access_method.inc @@ -20,11 +20,15 @@ SELECT pg_tde_is_encrypted('country_table'); SELECT pg_tde_is_encrypted('country_table_country_id_seq'); +SELECT pg_tde_is_encrypted('country_table_pkey'); + -- Try changing the encrypted table to an unencrypted table ALTER TABLE country_table SET access method heap; SELECT pg_tde_is_encrypted('country_table_country_id_seq'); +SELECT pg_tde_is_encrypted('country_table_pkey'); + -- Insert some more data INSERT INTO country_table (country_name, continent) VALUES ('France', 'Europe'), @@ -36,6 +40,8 @@ SELECT pg_tde_is_encrypted('country_table'); SELECT pg_tde_is_encrypted('country_table_country_id_seq'); +SELECT pg_tde_is_encrypted('country_table_pkey'); + -- Change it back to encrypted ALTER TABLE country_table SET access method :tde_am; @@ -48,6 +54,8 @@ SELECT pg_tde_is_encrypted('country_table'); SELECT pg_tde_is_encrypted('country_table_country_id_seq'); +SELECT pg_tde_is_encrypted('country_table_pkey'); + ALTER TABLE country_table ADD y text; SELECT pg_tde_is_encrypted(('pg_toast.pg_toast_' || 'country_table'::regclass::oid)::regclass); diff --git a/contrib/pg_tde/src/include/pg_tde_event_capture.h b/contrib/pg_tde/src/include/pg_tde_event_capture.h index d22ddc7fe41..bce7c2135da 100644 --- a/contrib/pg_tde/src/include/pg_tde_event_capture.h +++ b/contrib/pg_tde/src/include/pg_tde_event_capture.h @@ -18,8 +18,9 @@ typedef struct TdeCreateEvent * contains InvalidOid */ RangeVar *relation; /* Reference to the parsed relation from * create statement */ - bool alterSequenceMode; /* true when alter sequence is executed by - * pg_tde */ + bool alterAccessMethodMode; /* during ALTER ... SET ACCESS METHOD, + * new file permissions shouldn't be + * based on earlier encryption status. */ } TdeCreateEvent; extern TdeCreateEvent *GetCurrentTdeCreateEvent(void); diff --git a/contrib/pg_tde/src/pg_tde_event_capture.c b/contrib/pg_tde/src/pg_tde_event_capture.c index 3a03eba5b68..0dd403d58d0 100644 --- a/contrib/pg_tde/src/pg_tde_event_capture.c +++ b/contrib/pg_tde/src/pg_tde_event_capture.c @@ -187,6 +187,7 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) tdeCurrentCreateEvent.relation = stmt->relation; tdeCurrentCreateEvent.baseTableOid = relationId; + tdeCurrentCreateEvent.alterAccessMethodMode = true; checkEncryptionClause(accessMethod); alterSetAccessMethod = true; @@ -240,12 +241,18 @@ pg_tde_ddl_command_end_capture(PG_FUNCTION_ARGS) { #ifdef PERCONA_EXT + EventTriggerData *trigdata; + Node *parsetree; + + trigdata = (EventTriggerData *) fcinfo->context; + parsetree = trigdata->parsetree; + /* Ensure this function is being called as an event trigger */ if (!CALLED_AS_EVENT_TRIGGER(fcinfo)) /* internal error */ ereport(ERROR, (errmsg("Function can only be fired by event trigger manager"))); - if (alterSetAccessMethod && !tdeCurrentCreateEvent.alterSequenceMode) + if (IsA(parsetree, AlterTableStmt) && tdeCurrentCreateEvent.alterAccessMethodMode) { /* * sequences are not updated automatically call a helper function that @@ -267,9 +274,9 @@ pg_tde_ddl_command_end_capture(PG_FUNCTION_ARGS) args[0] = ObjectIdGetDatum(tdeCurrentCreateEvent.baseTableOid); nulls[0] = ' '; - tdeCurrentCreateEvent.alterSequenceMode = true; ret = SPI_execute_plan(plan, args, nulls, false, 0); - tdeCurrentCreateEvent.alterSequenceMode = false; + + tdeCurrentCreateEvent.alterAccessMethodMode = false; SPI_finish(); @@ -295,6 +302,7 @@ reset_current_tde_create_event(void) tdeCurrentCreateEvent.baseTableOid = InvalidOid; tdeCurrentCreateEvent.relation = NULL; alterSetAccessMethod = false; + tdeCurrentCreateEvent.alterAccessMethodMode = false; } static Oid diff --git a/contrib/pg_tde/src/smgr/pg_tde_smgr.c b/contrib/pg_tde/src/smgr/pg_tde_smgr.c index 650b1bf52fe..352a091ad35 100644 --- a/contrib/pg_tde/src/smgr/pg_tde_smgr.c +++ b/contrib/pg_tde/src/smgr/pg_tde_smgr.c @@ -238,7 +238,7 @@ tde_mdcreate(RelFileLocator relold, SMgrRelation reln, ForkNumber forknum, bool * Later calls then decide to encrypt or not based on the existence of the * key */ - key = tde_smgr_get_key(reln, event->alterSequenceMode ? NULL : &relold, true); + key = tde_smgr_get_key(reln, event->alterAccessMethodMode ? NULL : &relold, true); if (key) { From 0590dcbb13d31896df63ac3550a37885a5f4251a Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Mon, 10 Mar 2025 15:07:47 +0100 Subject: [PATCH 12/31] PG-1414 Fix reference to which fucntion to call in error message The error message when we have no principal key for the WAL encryption referred to the wrong function. --- contrib/pg_tde/src/access/pg_tde_tdemap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pg_tde/src/access/pg_tde_tdemap.c b/contrib/pg_tde/src/access/pg_tde_tdemap.c index 82188fe3e10..b4ecf7114fd 100644 --- a/contrib/pg_tde/src/access/pg_tde_tdemap.c +++ b/contrib/pg_tde/src/access/pg_tde_tdemap.c @@ -266,7 +266,7 @@ pg_tde_create_wal_key(InternalKey *rel_key_data, const RelFileLocator *newrlocat if (principal_key == NULL) { ereport(ERROR, - (errmsg("failed to retrieve principal key. Create one using pg_tde_set_principal_key before using encrypted WAL."))); + (errmsg("failed to retrieve principal key. Create one using pg_tde_set_server_principal_key before using encrypted WAL."))); return; } From 85a539f93be72cdd5b5f081154ee263c6592e25e Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Fri, 7 Mar 2025 22:28:21 +0000 Subject: [PATCH 13/31] PG-1417: Extension startup failure should be a fatal error Earlier we modified the shared_preload_libraries check to a normal error instead of fatal, to avoid the server closing the connection on error. This seemed like a change that makes the extension a bit more user friendly. Unfortunately this has an unforeseen sideeffect of the pg_dump + pg_restore/psql combo possibly restoring a dump of an encrypted database in a possibly unencrypted version. This is because: * pg_dump doesn't use the `USING tde_heap` clause for `CREATE TABLE`, it sets the `default_table_access_method` instead * both psql and pg_restore continue on errors by default, unless the -e command line argument is specified. In practice this means that even if the `SET default_table_access_method` call fails, the script continues, and creates tables using the "normal default", heap. Fix: revert the change, the extension startup failure should be a FATAL instead of a simple ERROR. --- contrib/pg_tde/src/pg_tde.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/contrib/pg_tde/src/pg_tde.c b/contrib/pg_tde/src/pg_tde.c index b42876a82ca..d89261e9328 100644 --- a/contrib/pg_tde/src/pg_tde.c +++ b/contrib/pg_tde/src/pg_tde.c @@ -111,7 +111,14 @@ _PG_init(void) { if (!process_shared_preload_libraries_in_progress) { - elog(ERROR, "pg_tde can only be loaded at server startup. Restart required."); + /* + * psql/pg_restore continue on error by default, and change access + * methods using set default_table_access_method. This error needs to + * be FATAL and close the connection, otherwise these tools will + * continue execution and create unencrypted tables when the intention + * was to make them encrypted. + */ + elog(FATAL, "pg_tde can only be loaded at server startup. Restart required."); return; } From dd13758f61a7706f55bac308cabc8606f527e286 Mon Sep 17 00:00:00 2001 From: Naeem Akhter Date: Wed, 12 Mar 2025 03:24:26 +0500 Subject: [PATCH 14/31] Added missing 'key' word at couple of places. And trimmed extra space at some line endings. --- contrib/pg_tde/documentation/docs/tde.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/tde.md b/contrib/pg_tde/documentation/docs/tde.md index a33ea97fa2f..7621753fc75 100644 --- a/contrib/pg_tde/documentation/docs/tde.md +++ b/contrib/pg_tde/documentation/docs/tde.md @@ -7,7 +7,7 @@ Transparent Data Encryption is a technology to protect data at rest. The encrypt To encrypt the data, two types of keys are used: * Internal encryption keys to encrypt user data. They are stored internally, near the data that they encrypt. -* The principal key to encrypt database keys. It is kept separately from the database keys and is managed externally in the key management store. +* The principal key to encrypt database keys. It is kept separately from the database keys and is managed externally in the key management store. You have the following options to store and manage principal keys externally: @@ -18,11 +18,11 @@ The encryption process is the following: ![image](_images/tde-flow.png) -When a user creates an encrypted table using `pg_tde`, a new random key is generated for that table using the AES128 (AES-ECB) cipher algorithm. This key is used to encrypt all data the user inserts in that table. Eventually the encrypted data gets stored in the underlying storage. +When a user creates an encrypted table using `pg_tde`, a new random key is generated for that table using the AES128 (AES-ECB) cipher algorithm. This key is used to encrypt all data the user inserts in that table. Eventually the encrypted data gets stored in the underlying storage. -The table itself is encrypted using the principal key. The principal key is stored externally in the key management store. +The table key itself is encrypted using the principal key. The principal key is stored externally in the key management store. -Similarly when the user queries the encrypted table, the principal key is retrieved from the key store to decrypt the table. Then the same unique internal key for that table is used to decrypt the data, and unencrypted data gets returned to the user. So, effectively, every TDE table has a unique key, and each table key is encrypted using the principal key. +Similarly when the user queries the encrypted table, the principal key is retrieved from the key store to decrypt the table key. Then the same unique internal key for that table is used to decrypt the data, and unencrypted data gets returned to the user. So, effectively, every TDE table has a unique key, and each table key is encrypted using the principal key. ## Why do you need TDE? From 6d28c6323bd878a2a05e31ca2d7638b56bb61487 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Tue, 11 Mar 2025 10:38:18 +0000 Subject: [PATCH 15/31] PG-1451: Event trigger ddl_end doesn't fire on failure Our logic expected the start/end triggers to fire always in pairs, always one end trigger for one start trigger. This is not the case, and it is clearly documented in the postgres documentation: end triggers are not executed on failed statements. This completely broke the internal logic for encryption, and it never worked after an error, requiring restarting the connection. Fix: instead of relying on start-end pairs, now we check the current transaction id both in the event trigger start function, and during file creation. If the transaction changed during the setup of the current event trigger data, we reset it. There's also some additional logic around when we should and when we can't start transactions: event triggers usually run before statements start up transactions, and some statements, such as DROP CONCURRENT / REINDEX CONCURRENT only work when theres no existing transaction. --- contrib/pg_tde/expected/no_provider_error.out | 8 +++ contrib/pg_tde/meson.build | 1 + contrib/pg_tde/sql/no_provider_error.sql | 11 +++ .../pg_tde/src/include/pg_tde_event_capture.h | 5 ++ contrib/pg_tde/src/pg_tde_event_capture.c | 71 ++++++++++++++----- contrib/pg_tde/src/smgr/pg_tde_smgr.c | 8 +++ 6 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 contrib/pg_tde/expected/no_provider_error.out create mode 100644 contrib/pg_tde/sql/no_provider_error.sql diff --git a/contrib/pg_tde/expected/no_provider_error.out b/contrib/pg_tde/expected/no_provider_error.out new file mode 100644 index 00000000000..e4eb1693d21 --- /dev/null +++ b/contrib/pg_tde/expected/no_provider_error.out @@ -0,0 +1,8 @@ +CREATE EXTENSION pg_tde; +-- should fail +CREATE TABLE t1 (n INT) USING tde_heap; +ERROR: failed to retrieve principal key. Create one using pg_tde_set_principal_key before using encrypted tables. +-- should work +CREATE TABLE t2 (n INT) USING heap; +DROP TABLE t2; +DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/meson.build b/contrib/pg_tde/meson.build index b7ab7b3e8f3..54f2fcce29c 100644 --- a/contrib/pg_tde/meson.build +++ b/contrib/pg_tde/meson.build @@ -142,6 +142,7 @@ if get_variable('percona_ext', false) 'kmip_test', 'alter_index', 'merge_join', + 'no_provider_error', ] tap_tests += [ diff --git a/contrib/pg_tde/sql/no_provider_error.sql b/contrib/pg_tde/sql/no_provider_error.sql new file mode 100644 index 00000000000..abe2e18cbcc --- /dev/null +++ b/contrib/pg_tde/sql/no_provider_error.sql @@ -0,0 +1,11 @@ +CREATE EXTENSION pg_tde; + +-- should fail +CREATE TABLE t1 (n INT) USING tde_heap; + +-- should work +CREATE TABLE t2 (n INT) USING heap; + +DROP TABLE t2; + +DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/src/include/pg_tde_event_capture.h b/contrib/pg_tde/src/include/pg_tde_event_capture.h index bce7c2135da..6ddf64e420b 100644 --- a/contrib/pg_tde/src/include/pg_tde_event_capture.h +++ b/contrib/pg_tde/src/include/pg_tde_event_capture.h @@ -9,9 +9,12 @@ #include "postgres.h" #include "nodes/parsenodes.h" +#include "access/transam.h" typedef struct TdeCreateEvent { + FullTransactionId tid; /* transaction id of the last event trigger, + * or 0 */ bool encryptMode; /* true when the table uses encryption */ Oid baseTableOid; /* Oid of table on which index is being * created on. For create table statement this @@ -25,4 +28,6 @@ typedef struct TdeCreateEvent extern TdeCreateEvent *GetCurrentTdeCreateEvent(void); +extern void validateCurrentEventTriggerState(bool mightStartTransaction); + #endif diff --git a/contrib/pg_tde/src/pg_tde_event_capture.c b/contrib/pg_tde/src/pg_tde_event_capture.c index 0dd403d58d0..cf9999f548c 100644 --- a/contrib/pg_tde/src/pg_tde_event_capture.c +++ b/contrib/pg_tde/src/pg_tde_event_capture.c @@ -31,13 +31,10 @@ #include "executor/spi.h" /* Global variable that gets set at ddl start and cleard out at ddl end*/ -TdeCreateEvent tdeCurrentCreateEvent = {.relation = NULL}; +TdeCreateEvent tdeCurrentCreateEvent = {.tid = {.value = 0},.relation = NULL}; bool alterSetAccessMethod = false; -int event_trigger_level = 0; - - static void reset_current_tde_create_event(void); static Oid get_tde_table_am_oid(void); @@ -88,6 +85,23 @@ checkEncryptionClause(const char *accessMethod) } } +void +validateCurrentEventTriggerState(bool mightStartTransaction) +{ + FullTransactionId tid = mightStartTransaction ? GetCurrentFullTransactionId() : GetCurrentFullTransactionIdIfAny(); + + if (RecoveryInProgress()) + { + reset_current_tde_create_event(); + return; + } + if (tdeCurrentCreateEvent.tid.value != InvalidFullTransactionId.value && tid.value != tdeCurrentCreateEvent.tid.value) + { + /* There was a failed query, end event trigger didn't execute */ + reset_current_tde_create_event(); + } +} + /* * pg_tde_ddl_command_start_capture is an event trigger function triggered * at the start of any DDL command execution. @@ -108,8 +122,6 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) EventTriggerData *trigdata; Node *parsetree; - event_trigger_level++; - /* Ensure this function is being called as an event trigger */ if (!CALLED_AS_EVENT_TRIGGER(fcinfo)) /* internal error */ ereport(ERROR, @@ -118,16 +130,14 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) trigdata = (EventTriggerData *) fcinfo->context; parsetree = trigdata->parsetree; - if (event_trigger_level == 1) - { - reset_current_tde_create_event(); - } - if (IsA(parsetree, IndexStmt)) { IndexStmt *stmt = (IndexStmt *) parsetree; Oid relationId = RangeVarGetRelid(stmt->relation, NoLock, true); + validateCurrentEventTriggerState(true); + tdeCurrentCreateEvent.tid = GetCurrentFullTransactionId(); + tdeCurrentCreateEvent.baseTableOid = relationId; tdeCurrentCreateEvent.relation = stmt->relation; @@ -158,6 +168,10 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) CreateStmt *stmt = (CreateStmt *) parsetree; const char *accessMethod = stmt->accessMethod; + validateCurrentEventTriggerState(true); + tdeCurrentCreateEvent.tid = GetCurrentFullTransactionId(); + + tdeCurrentCreateEvent.relation = stmt->relation; checkEncryptionClause(accessMethod); @@ -167,6 +181,9 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) CreateTableAsStmt *stmt = (CreateTableAsStmt *) parsetree; const char *accessMethod = stmt->into->accessMethod; + validateCurrentEventTriggerState(true); + tdeCurrentCreateEvent.tid = GetCurrentFullTransactionId(); + tdeCurrentCreateEvent.relation = stmt->into->rel; checkEncryptionClause(accessMethod); @@ -177,6 +194,9 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) ListCell *lcmd; Oid relationId = RangeVarGetRelid(stmt->relation, NoLock, true); + validateCurrentEventTriggerState(true); + tdeCurrentCreateEvent.tid = GetCurrentFullTransactionId(); + foreach(lcmd, stmt->cmds) { AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lcmd); @@ -226,7 +246,19 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) } } } - + } + else + { + if (!tdeCurrentCreateEvent.alterAccessMethodMode) + { + /* + * Any other type of statement doesn't need TDE mode, except + * during alter access method. To make sure that we have no + * leftover setting from a previous error or something, we just + * reset the status here. + */ + reset_current_tde_create_event(); + } } #endif PG_RETURN_NULL(); @@ -286,10 +318,16 @@ pg_tde_ddl_command_end_capture(PG_FUNCTION_ARGS) } } - event_trigger_level--; - - /* All we need to do is to clear the event state */ - reset_current_tde_create_event(); + /* + * All we need to do is to clear the event state. Except when we are in + * alter access method mode, because during that, we have multiple nested + * event trigger running. Reset should only be called in the end, when it + * is set to false. + */ + if (!tdeCurrentCreateEvent.alterAccessMethodMode) + { + reset_current_tde_create_event(); + } #endif PG_RETURN_NULL(); @@ -301,6 +339,7 @@ reset_current_tde_create_event(void) tdeCurrentCreateEvent.encryptMode = false; tdeCurrentCreateEvent.baseTableOid = InvalidOid; tdeCurrentCreateEvent.relation = NULL; + tdeCurrentCreateEvent.tid = InvalidFullTransactionId; alterSetAccessMethod = false; tdeCurrentCreateEvent.alterAccessMethodMode = false; } diff --git a/contrib/pg_tde/src/smgr/pg_tde_smgr.c b/contrib/pg_tde/src/smgr/pg_tde_smgr.c index 352a091ad35..6104b9e49b3 100644 --- a/contrib/pg_tde/src/smgr/pg_tde_smgr.c +++ b/contrib/pg_tde/src/smgr/pg_tde_smgr.c @@ -226,6 +226,14 @@ tde_mdcreate(RelFileLocator relold, SMgrRelation reln, ForkNumber forknum, bool InternalKey *key; TdeCreateEvent *event = GetCurrentTdeCreateEvent(); + /* + * Make sure that even if a statement failed, and an event trigger end + * trigger didn't fire, we don't accidentaly create encrypted files when + * we don't have to. event above is a pointer, so it will reflect the + * correct state even if this changes it. + */ + validateCurrentEventTriggerState(false); + /* * This is the only function that gets called during actual CREATE * TABLE/INDEX (EVENT TRIGGER) From 18e2e606cae2045e5ae036717e00fab157d894b4 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoi Date: Wed, 12 Mar 2025 16:36:32 +0200 Subject: [PATCH 16/31] PG-1413: fix locking for WAL key update (#124) Key rotation creates a new _dat file and swaps with the existing one after all changes have been made. Therefore, the lock should be held before opening the descriptor while updating a WAL key's LSN. What could have happened (and had happened) before this commit: 1. `pg_tde_wal_last_key_set_lsn()` opened a _dat descriptor 2. `pg_tde_perform_rotate_key()` aquires the lock 3. `pg_tde_perform_rotate_key()` performs a rotation and rewrites the _dat with a new copy (with a new fd). 4. `pg_tde_perform_rotate_key()` releases the lock 5. `pg_tde_wal_last_key_set_lsn()` acquires the lock 6. `pg_tde_wal_last_key_set_lsn()` writes changes to the old fd (into the void). As a result, the LSN of the key in the proper file remains unchanged. --- contrib/pg_tde/src/access/pg_tde_tdemap.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/pg_tde/src/access/pg_tde_tdemap.c b/contrib/pg_tde/src/access/pg_tde_tdemap.c index b4ecf7114fd..4d0dc5ed236 100644 --- a/contrib/pg_tde/src/access/pg_tde_tdemap.c +++ b/contrib/pg_tde/src/access/pg_tde_tdemap.c @@ -969,6 +969,8 @@ pg_tde_wal_last_key_set_lsn(XLogRecPtr lsn, const char *keyfile_path) prev_key_pos; InternalKey prev_key; + LWLockAcquire(lock_pk, LW_EXCLUSIVE); + fd = BasicOpenFile(keyfile_path, O_RDWR | PG_BINARY); if (fd < 0) { @@ -981,7 +983,6 @@ pg_tde_wal_last_key_set_lsn(XLogRecPtr lsn, const char *keyfile_path) last_key_idx = ((lseek(fd, 0, SEEK_END) - TDE_FILE_HEADER_SIZE) / INTERNAL_KEY_DAT_LEN) - 1; write_pos = TDE_FILE_HEADER_SIZE + (last_key_idx * INTERNAL_KEY_DAT_LEN) + offsetof(InternalKey, start_lsn); - LWLockAcquire(lock_pk, LW_EXCLUSIVE); /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ if (pg_pwrite(fd, &lsn, sizeof(XLogRecPtr), write_pos) != sizeof(XLogRecPtr)) { From 084c8c5c38d9bcd77b467eb9578c183e0d06f7df Mon Sep 17 00:00:00 2001 From: Naeem Akhter Date: Thu, 13 Mar 2025 01:09:49 +0500 Subject: [PATCH 17/31] Fixed postgres service name for debain/ubuntu on pg_tde setup doc. --- contrib/pg_tde/documentation/docs/setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pg_tde/documentation/docs/setup.md b/contrib/pg_tde/documentation/docs/setup.md index 156a45fdeea..5963ead19fe 100644 --- a/contrib/pg_tde/documentation/docs/setup.md +++ b/contrib/pg_tde/documentation/docs/setup.md @@ -19,7 +19,7 @@ Load the `pg_tde` at startup time. The extension requires additional shared memo * On Debian and Ubuntu: ```sh - sudo systemctl restart postgresql-17 + sudo systemctl restart postgresql@17-main ``` * On RHEL and derivatives From f772b7661fc9cb345b4b6cc06a7f93ffb5f33e60 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoi Date: Thu, 13 Mar 2025 18:29:27 +0200 Subject: [PATCH 18/31] Fix WAL encryption buffer size (#130) 1. We used to base the enc buffer size on `wal_buffers`. But walsender has its own constant send size, which might be bigger than wal_buffers and lead to an overflow and the corruption of neighboring objects in the shared memory. This commit ensures the enc buffer is big enough for wal_buffers and walsender. 2. It also fixes the enc buffer alignment. For PG-1397, PG-1037 --- .../pg_tde/src/access/pg_tde_xlog_encrypt.c | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c b/contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c index 219e41b3409..ecca6f66ab5 100644 --- a/contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c +++ b/contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c @@ -43,6 +43,14 @@ static const XLogSmgr tde_xlog_smgr = { static void SetXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, char *iv_prefix); #ifndef FRONTEND + +/* + * Must be the same as in replication/walsender.c + * + * This is used to calculate the encryption buffer size. + */ +#define MAX_SEND_SIZE (XLOG_BLCKSZ * 16) + static ssize_t TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset, TimeLineID tli, XLogSegNo segno); @@ -104,7 +112,7 @@ TDEXLogEncryptBuffSize(void) int xbuffers; xbuffers = (XLOGbuffers == -1) ? XLOGChooseNumBuffers() : XLOGbuffers; - return (Size) XLOG_BLCKSZ * xbuffers; + return Max(MAX_SEND_SIZE, mul_size(XLOG_BLCKSZ, xbuffers)); } Size @@ -112,10 +120,11 @@ TDEXLogEncryptStateSize(void) { Size sz; - sz = TYPEALIGN(PG_IO_ALIGN_SIZE, TDEXLogEncryptBuffSize()); - sz = add_size(sz, sizeof(EncryptionStateData)); + sz = sizeof(EncryptionStateData); + sz = add_size(sz, TDEXLogEncryptBuffSize()); + sz = add_size(sz, PG_IO_ALIGN_SIZE); - return MAXALIGN(sz); + return sz; } /* @@ -143,9 +152,14 @@ TDEXLogShmemInit(void) TDEXLogEncryptStateSize(), &foundBuf); - allocptr = ((char *) EncryptionState) + TYPEALIGN(PG_IO_ALIGN_SIZE, sizeof(EncryptionStateData)); + memset(EncryptionState, 0, sizeof(EncryptionStateData)); + + allocptr = ((char *) EncryptionState) + sizeof(EncryptionStateData); + allocptr = (char *) TYPEALIGN(PG_IO_ALIGN_SIZE, allocptr); EncryptionState->segBuf = allocptr; + Assert((char *) EncryptionState + TDEXLogEncryptStateSize() >= (char *) EncryptionState->segBuf + TDEXLogEncryptBuffSize()); + pg_atomic_init_u64(&EncryptionState->enc_key_lsn, 0); elog(DEBUG1, "pg_tde: initialized encryption buffer %lu bytes", TDEXLogEncryptStateSize()); @@ -162,6 +176,8 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset, InternalKey *key = &EncryptionKey; char *enc_buff = EncryptionState->segBuf; + Assert(count <= TDEXLogEncryptBuffSize()); + #ifdef TDE_XLOG_DEBUG elog(DEBUG1, "write encrypted WAL, size: %lu, offset: %ld [%lX], seg: %X/%X, key_start_lsn: %X/%X", count, offset, offset, LSN_FORMAT_ARGS(segno), LSN_FORMAT_ARGS(key->start_lsn)); From 430bc0f95262dd5b0422e77a0661483a481cb747 Mon Sep 17 00:00:00 2001 From: Shahid Ullah <32063351+shahidullah79@users.noreply.github.com> Date: Fri, 14 Mar 2025 02:00:01 +0500 Subject: [PATCH 19/31] [PG-1473] Fixed viewer permission issue for pg_tde_verify_principal_key (#134) --- contrib/pg_tde/pg_tde--1.0-rc.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/pg_tde/pg_tde--1.0-rc.sql b/contrib/pg_tde/pg_tde--1.0-rc.sql index 13f544372bf..4657863607c 100644 --- a/contrib/pg_tde/pg_tde--1.0-rc.sql +++ b/contrib/pg_tde/pg_tde--1.0-rc.sql @@ -642,6 +642,7 @@ BEGIN EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_principal_key_info() TO %I', target_role); EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_global_principal_key_info() TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_verify_principal_key() TO %I', target_role); END; $$; @@ -722,6 +723,7 @@ BEGIN EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_principal_key_info() FROM %I', target_role); EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_global_principal_key_info() FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_verify_principal_key() FROM %I', target_role); END; $$; From 5ba2bf7d6a5e98ced3963d6e88b6fe1f98f289c6 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Wed, 12 Mar 2025 20:19:06 +0000 Subject: [PATCH 20/31] PG-1450: Ignore empty names when looking for providers to modify Modification looks are either based on provider id or name. Delete functionality uses id, and passes an empty name. This failed if an earlier record was already deleted, as the lookup found the record with the empty name instead of the correct id. As empty names can only happen with deleted records, and we never want to delete rerord twice, the function now ignores those. --- .../pg_tde/expected/default_principal_key.out | 3 +- .../pg_tde/expected/delete_key_provider.out | 74 +++++++++++++++++++ contrib/pg_tde/expected/key_provider.out | 3 +- contrib/pg_tde/meson.build | 1 + contrib/pg_tde/sql/delete_key_provider.sql | 20 +++++ contrib/pg_tde/src/catalog/tde_keyring.c | 2 +- 6 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 contrib/pg_tde/expected/delete_key_provider.out create mode 100644 contrib/pg_tde/sql/delete_key_provider.sql diff --git a/contrib/pg_tde/expected/default_principal_key.out b/contrib/pg_tde/expected/default_principal_key.out index dfea64ae262..c8f28f2d5e4 100644 --- a/contrib/pg_tde/expected/default_principal_key.out +++ b/contrib/pg_tde/expected/default_principal_key.out @@ -17,9 +17,8 @@ ERROR: Can't delete a provider which is currently in use SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); id | provider_name ----+--------------- - -2 | file-keyring2 -3 | file-provider -(2 rows) +(1 row) -- Should fail: no principal key for the database yet SELECT key_provider_id, key_provider_name, principal_key_name diff --git a/contrib/pg_tde/expected/delete_key_provider.out b/contrib/pg_tde/expected/delete_key_provider.out new file mode 100644 index 00000000000..f4f4ed109db --- /dev/null +++ b/contrib/pg_tde/expected/delete_key_provider.out @@ -0,0 +1,74 @@ +CREATE EXTENSION IF NOT EXISTS pg_tde; +SELECT * FROM pg_tde_principal_key_info(); +ERROR: Principal key does not exists for the database +HINT: Use set_principal_key interface to set the principal key +SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_key_provider_file +------------------------------ + 1 +(1 row) + +SELECT * FROM pg_tde_list_all_key_providers(); + id | provider_name | provider_type | options +----+---------------+---------------+------------------------------------------------------------ + 1 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} +(1 row) + +SELECT pg_tde_delete_key_provider('file-provider'); + pg_tde_delete_key_provider +---------------------------- + +(1 row) + +SELECT * FROM pg_tde_list_all_key_providers(); + id | provider_name | provider_type | options +----+---------------+---------------+--------- +(0 rows) + +SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_key_provider_file +------------------------------ + 2 +(1 row) + +SELECT * FROM pg_tde_list_all_key_providers(); + id | provider_name | provider_type | options +----+---------------+---------------+------------------------------------------------------------ + 2 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} +(1 row) + +SELECT pg_tde_delete_key_provider('file-provider'); + pg_tde_delete_key_provider +---------------------------- + +(1 row) + +SELECT * FROM pg_tde_list_all_key_providers(); + id | provider_name | provider_type | options +----+---------------+---------------+--------- +(0 rows) + +SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_key_provider_file +------------------------------ + 3 +(1 row) + +SELECT * FROM pg_tde_list_all_key_providers(); + id | provider_name | provider_type | options +----+---------------+---------------+------------------------------------------------------------ + 3 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} +(1 row) + +SELECT pg_tde_delete_key_provider('file-provider'); + pg_tde_delete_key_provider +---------------------------- + +(1 row) + +SELECT * FROM pg_tde_list_all_key_providers(); + id | provider_name | provider_type | options +----+---------------+---------------+--------- +(0 rows) + +DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/expected/key_provider.out b/contrib/pg_tde/expected/key_provider.out index 16a6d6ed92e..86ee0ca3e6f 100644 --- a/contrib/pg_tde/expected/key_provider.out +++ b/contrib/pg_tde/expected/key_provider.out @@ -159,7 +159,6 @@ SELECT pg_tde_delete_global_key_provider('file-keyring2'); SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); id | provider_name ----+--------------- - -2 | file-keyring2 -(1 row) +(0 rows) DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/meson.build b/contrib/pg_tde/meson.build index 54f2fcce29c..19d55ef5e1d 100644 --- a/contrib/pg_tde/meson.build +++ b/contrib/pg_tde/meson.build @@ -134,6 +134,7 @@ if get_variable('percona_ext', false) 'test_issue_153_fix', 'multi_insert', 'keyprovider_dependency', + 'delete_key_provider', 'trigger_on_view', 'change_access_method', 'insert_update_delete', diff --git a/contrib/pg_tde/sql/delete_key_provider.sql b/contrib/pg_tde/sql/delete_key_provider.sql new file mode 100644 index 00000000000..431c97d6cc8 --- /dev/null +++ b/contrib/pg_tde/sql/delete_key_provider.sql @@ -0,0 +1,20 @@ +CREATE EXTENSION IF NOT EXISTS pg_tde; + +SELECT * FROM pg_tde_principal_key_info(); + +SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); +SELECT * FROM pg_tde_list_all_key_providers(); +SELECT pg_tde_delete_key_provider('file-provider'); +SELECT * FROM pg_tde_list_all_key_providers(); + +SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); +SELECT * FROM pg_tde_list_all_key_providers(); +SELECT pg_tde_delete_key_provider('file-provider'); +SELECT * FROM pg_tde_list_all_key_providers(); + +SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); +SELECT * FROM pg_tde_list_all_key_providers(); +SELECT pg_tde_delete_key_provider('file-provider'); +SELECT * FROM pg_tde_list_all_key_providers(); + +DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/src/catalog/tde_keyring.c b/contrib/pg_tde/src/catalog/tde_keyring.c index 5e415fbf533..94038bec140 100644 --- a/contrib/pg_tde/src/catalog/tde_keyring.c +++ b/contrib/pg_tde/src/catalog/tde_keyring.c @@ -444,7 +444,7 @@ write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, seek_pos = before_pos; break; } - if (strcmp(existing_provider.provider_name, provider->provider_name) == 0) + if (strlen(existing_provider.provider_name) > 0 && strcmp(existing_provider.provider_name, provider->provider_name) == 0) { if (error_if_exists) { From 116bf245f7be7b2afe4fbf4d78dc73729df9e9df Mon Sep 17 00:00:00 2001 From: Naeem Akhter Date: Fri, 14 Mar 2025 03:02:38 +0500 Subject: [PATCH 21/31] Added note to setup (doc) page regarding composition of client key. --- contrib/pg_tde/documentation/docs/setup.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/pg_tde/documentation/docs/setup.md b/contrib/pg_tde/documentation/docs/setup.md index 5963ead19fe..9a7994188c3 100644 --- a/contrib/pg_tde/documentation/docs/setup.md +++ b/contrib/pg_tde/documentation/docs/setup.md @@ -64,6 +64,8 @@ Load the `pg_tde` at startup time. The extension requires additional shared memo * `server-certificate` is the path to the certificate file for the KMIP server. * `client key` is the path to the client key. + :material-information: Warning: Note that pg_tde_add_global_key_provider_kmip currently accepts only a combined client key + client certificate for the last parameter of this function named as `client key`. + :material-information: Warning: This example is for testing purposes only: ``` From c72b4c361867bf6a52fd2834eb2a105311a66079 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Thu, 13 Mar 2025 22:57:43 +0000 Subject: [PATCH 22/31] PG-1474: Properly set the key name in vault/kmip getters Vault and KMIP providers forgot to set principal key name in the getter which returned the key from the server, resulting in a corrupted key reference when the key already existed on the remote server. --- contrib/pg_tde/expected/vault_v2_test.out | 19 ++++++++++++ .../pg_tde/expected/vault_v2_test_basic.out | 31 +++++++++++++++---- contrib/pg_tde/sql/vault_v2_test.inc | 7 +++++ contrib/pg_tde/src/keyring/keyring_kmip.c | 2 ++ contrib/pg_tde/src/keyring/keyring_vault.c | 2 ++ 5 files changed, 55 insertions(+), 6 deletions(-) diff --git a/contrib/pg_tde/expected/vault_v2_test.out b/contrib/pg_tde/expected/vault_v2_test.out index 08f395fc86a..85701b0a4c5 100644 --- a/contrib/pg_tde/expected/vault_v2_test.out +++ b/contrib/pg_tde/expected/vault_v2_test.out @@ -29,6 +29,25 @@ SELECT pg_tde_set_principal_key('vault-v2-principal-key','vault-v2'); t (1 row) +SELECT pg_tde_set_principal_key('vault-v2-principal-key-2','vault-v2'); + pg_tde_set_principal_key +-------------------------- + t +(1 row) + +SELECT pg_tde_set_principal_key('vault-v2-principal-key','vault-v2'); + pg_tde_set_principal_key +-------------------------- + t +(1 row) + +SELECT key_provider_id, key_provider_name, principal_key_name + FROM pg_tde_principal_key_info(); + key_provider_id | key_provider_name | principal_key_name +-----------------+-------------------+------------------------ + 2 | vault-v2 | vault-v2-principal-key +(1 row) + CREATE TABLE test_enc( id SERIAL, k INTEGER DEFAULT '0' NOT NULL, diff --git a/contrib/pg_tde/expected/vault_v2_test_basic.out b/contrib/pg_tde/expected/vault_v2_test_basic.out index f84a0a7c222..ec04768cb56 100644 --- a/contrib/pg_tde/expected/vault_v2_test_basic.out +++ b/contrib/pg_tde/expected/vault_v2_test_basic.out @@ -30,21 +30,40 @@ SELECT pg_tde_set_principal_key('vault-v2-principal-key','vault-v2'); t (1 row) +SELECT pg_tde_set_principal_key('vault-v2-principal-key-2','vault-v2'); + pg_tde_set_principal_key +-------------------------- + t +(1 row) + +SELECT pg_tde_set_principal_key('vault-v2-principal-key','vault-v2'); + pg_tde_set_principal_key +-------------------------- + t +(1 row) + +SELECT key_provider_id, key_provider_name, principal_key_name + FROM pg_tde_principal_key_info(); + key_provider_id | key_provider_name | principal_key_name +-----------------+-------------------+------------------------ + 2 | vault-v2 | vault-v2-principal-key +(1 row) + CREATE TABLE test_enc( id SERIAL, k INTEGER DEFAULT '0' NOT NULL, PRIMARY KEY (id) ) USING :tde_am; -psql:sql/vault_v2_test.inc:22: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. -psql:sql/vault_v2_test.inc:22: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/vault_v2_test.inc:29: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/vault_v2_test.inc:29: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. INSERT INTO test_enc (k) VALUES (1); -psql:sql/vault_v2_test.inc:24: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/vault_v2_test.inc:31: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. INSERT INTO test_enc (k) VALUES (2); -psql:sql/vault_v2_test.inc:25: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/vault_v2_test.inc:32: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. INSERT INTO test_enc (k) VALUES (3); -psql:sql/vault_v2_test.inc:26: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/vault_v2_test.inc:33: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. SELECT * from test_enc; -psql:sql/vault_v2_test.inc:28: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. +psql:sql/vault_v2_test.inc:35: WARNING: tde_heap_basic is deprecated, and will be removed in the next release. Please migrate tables to tde_heap. id | k ----+--- 1 | 1 diff --git a/contrib/pg_tde/sql/vault_v2_test.inc b/contrib/pg_tde/sql/vault_v2_test.inc index 0c715592875..69c65914339 100644 --- a/contrib/pg_tde/sql/vault_v2_test.inc +++ b/contrib/pg_tde/sql/vault_v2_test.inc @@ -15,6 +15,13 @@ CREATE TABLE test_enc( SELECT pg_tde_add_key_provider_vault_v2('vault-v2',:'root_token','http://127.0.0.1:8200','secret',NULL); SELECT pg_tde_set_principal_key('vault-v2-principal-key','vault-v2'); +SELECT pg_tde_set_principal_key('vault-v2-principal-key-2','vault-v2'); + +SELECT pg_tde_set_principal_key('vault-v2-principal-key','vault-v2'); + +SELECT key_provider_id, key_provider_name, principal_key_name + FROM pg_tde_principal_key_info(); + CREATE TABLE test_enc( id SERIAL, k INTEGER DEFAULT '0' NOT NULL, diff --git a/contrib/pg_tde/src/keyring/keyring_kmip.c b/contrib/pg_tde/src/keyring/keyring_kmip.c index 6c8e406214e..6125d66838c 100644 --- a/contrib/pg_tde/src/keyring/keyring_kmip.c +++ b/contrib/pg_tde/src/keyring/keyring_kmip.c @@ -264,6 +264,8 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, bool throw_error, return NULL; } + memset(key->name, 0, sizeof(key->name)); + memcpy(key->name, key_name, strnlen(key_name, sizeof(key->name) - 1)); memcpy(key->data.data, keyp, key->data.len); free(keyp); } diff --git a/contrib/pg_tde/src/keyring/keyring_vault.c b/contrib/pg_tde/src/keyring/keyring_vault.c index 3dd505034c6..ad0c454920f 100644 --- a/contrib/pg_tde/src/keyring/keyring_vault.c +++ b/contrib/pg_tde/src/keyring/keyring_vault.c @@ -283,6 +283,8 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, bool throw_error, #endif key = palloc(sizeof(KeyInfo)); + memset(key->name, 0, sizeof(key->name)); + memcpy(key->name, key_name, strnlen(key_name, sizeof(key->name) - 1)); key->data.len = pg_b64_decode(responseKey, strlen(responseKey), (char *) key->data.data, MAX_KEY_DATA_SIZE); if (key->data.len > MAX_KEY_DATA_SIZE) From fb5658242eed71b645768ec35ec13d3e4c8cc1e4 Mon Sep 17 00:00:00 2001 From: Naeem Akhter Date: Sat, 15 Mar 2025 03:34:30 +0500 Subject: [PATCH 23/31] PG-1484 Added a test case for pg_tde version check. --- contrib/pg_tde/Makefile | 3 ++- contrib/pg_tde/expected/version.out | 8 ++++++++ contrib/pg_tde/sql/version.sql | 3 +++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 contrib/pg_tde/expected/version.out create mode 100644 contrib/pg_tde/sql/version.sql diff --git a/contrib/pg_tde/Makefile b/contrib/pg_tde/Makefile index 2eb37751118..50c839e5b9a 100644 --- a/contrib/pg_tde/Makefile +++ b/contrib/pg_tde/Makefile @@ -24,7 +24,8 @@ vault_v2_test_basic \ alter_index_basic \ merge_join_basic \ tablespace_basic \ -relocate +relocate \ +version TAP_TESTS = 1 OBJS = src/encryption/enc_tde.o \ diff --git a/contrib/pg_tde/expected/version.out b/contrib/pg_tde/expected/version.out new file mode 100644 index 00000000000..fddbafca98f --- /dev/null +++ b/contrib/pg_tde/expected/version.out @@ -0,0 +1,8 @@ +CREATE EXTENSION pg_tde; +SELECT pg_tde_version(); + pg_tde_version +----------------- + pg_tde 1.0.0-rc +(1 row) + +DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/sql/version.sql b/contrib/pg_tde/sql/version.sql new file mode 100644 index 00000000000..bedfa170a9c --- /dev/null +++ b/contrib/pg_tde/sql/version.sql @@ -0,0 +1,3 @@ +CREATE EXTENSION pg_tde; +SELECT pg_tde_version(); +DROP EXTENSION pg_tde; From e21b7487623797dc39000d6b788580251d4b864e Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Sat, 15 Mar 2025 08:15:51 +0000 Subject: [PATCH 24/31] PG-1473: Also manage verify_global_principal_key permissions correctly --- contrib/pg_tde/pg_tde--1.0-rc.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/pg_tde/pg_tde--1.0-rc.sql b/contrib/pg_tde/pg_tde--1.0-rc.sql index 4657863607c..c73710ff1ea 100644 --- a/contrib/pg_tde/pg_tde--1.0-rc.sql +++ b/contrib/pg_tde/pg_tde--1.0-rc.sql @@ -643,6 +643,7 @@ BEGIN EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_principal_key_info() TO %I', target_role); EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_global_principal_key_info() TO %I', target_role); EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_verify_principal_key() TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_verify_global_principal_key() TO %I', target_role); END; $$; @@ -724,6 +725,7 @@ BEGIN EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_principal_key_info() FROM %I', target_role); EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_global_principal_key_info() FROM %I', target_role); EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_verify_principal_key() FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_verify_global_principal_key() FROM %I', target_role); END; $$; From 54ed4218cb4d05f190587148783fed1c7ddb9c33 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Sat, 15 Mar 2025 17:51:50 +0000 Subject: [PATCH 25/31] Added missing function to documentation --- contrib/pg_tde/documentation/docs/functions.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index 4524cf8ec53..f30929a5da3 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -203,7 +203,7 @@ Princial keys are stored on key providers by the name specified in this function ### pg_tde_set_principal_key -Creates or rotates the principal key for the current database using the specified key provider and name. +Creates or rotates the principal key for the current database using the specified database key provider and key name. ``` SELECT pg_tde_set_principal_key('name-of-the-principal-key','provider-name','ensure_new_key'); @@ -215,6 +215,20 @@ SELECT pg_tde_set_principal_key('name-of-the-principal-key','provider-name','ens If the provider already stores a key by that name, the function returns an error. * If set to `false`, an existing principal key may be reused. +### pg_tde_set_global_principal_key + +Creates or rotates the principal key for the current database using the specified global key provider and key name. + +``` +SELECT pg_tde_set_global_principal_key('name-of-the-principal-key','provider-name','ensure_new_key'); +``` + + The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: + +* If set to `true` (default), a new key must be unique. + If the provider already stores a key by that name, the function returns an error. +* If set to `false`, an existing principal key may be reused. + ### pg_tde_set_server_principal_key Creates or rotates the global principal key using the specified key provider. Use this function to set a principal key for WAL encryption. From cb06bea2537a7e9d354aeac0ddb24b3cddc4530f Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Fri, 14 Mar 2025 20:15:02 +0000 Subject: [PATCH 26/31] PG-1479,PG-1480: fix incorrect parameter order in default key rotation The new and old principal keys were switched for the rotate function, and as we do not have principal key validation for tdemap data, the function doesn't notice this. The problem also isn't visible until a server restart / new connection because of internal key caching, which means the SQL tests also missed to detect this. --- contrib/pg_tde/src/catalog/tde_principal_key.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pg_tde/src/catalog/tde_principal_key.c b/contrib/pg_tde/src/catalog/tde_principal_key.c index c30caf7e276..83d3d716966 100644 --- a/contrib/pg_tde/src/catalog/tde_principal_key.c +++ b/contrib/pg_tde/src/catalog/tde_principal_key.c @@ -1026,7 +1026,7 @@ pg_tde_rotate_default_key_for_database(TDEPrincipalKey *oldKey, TDEPrincipalKey newKey->keyInfo.databaseId = oldKey->keyInfo.databaseId; /* key rotation */ - is_rotated = pg_tde_perform_rotate_key(newKey, oldKey); + is_rotated = pg_tde_perform_rotate_key(oldKey, newKey); if (is_rotated && (!TDEisInGlobalSpace(newKey->keyInfo.databaseId))) { From d1eace54279550509d81dff64e245f2b1a2ba68c Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoi Date: Tue, 18 Mar 2025 14:34:39 +0200 Subject: [PATCH 27/31] Don't Xlog new use of default keys (#148) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We should not XLog a default key copy. It doesn’t make sense for reliability as we write results dirctly to the disk. And as for the replicas, they would just try to get a key, retrieve the default one and do copying on its own instead of reading that from the XLog. XLogging a new use of default keys creates the next issue: On server start, the WAL init tries to get a current WAL key to decide what to do next - create a new encrypted or unencrypted key, or do nothing - and for that it needs a principal key. During the GetPrincipalKey call, if there is only a default principal key, the server will create a new principal key for WAL (in this case) by copying the default key with the WAL Oid. Hence, create a new principal key with XLogInserts generated. Fixes PG-1476 --- contrib/pg_tde/src/catalog/tde_principal_key.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/contrib/pg_tde/src/catalog/tde_principal_key.c b/contrib/pg_tde/src/catalog/tde_principal_key.c index 83d3d716966..49d2c1ec29b 100644 --- a/contrib/pg_tde/src/catalog/tde_principal_key.c +++ b/contrib/pg_tde/src/catalog/tde_principal_key.c @@ -905,11 +905,6 @@ GetPrincipalKey(Oid dbOid, LWLockMode lockMode) create_principal_key_info(&newPrincipalKey->keyInfo); - /* XLog the new use of the default key */ - XLogBeginInsert(); - XLogRegisterData((char *) &newPrincipalKey->keyInfo, sizeof(TDEPrincipalKeyInfo)); - XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_PRINCIPAL_KEY); - push_principal_key_to_cache(newPrincipalKey); pfree(newPrincipalKey); From e3cd93bd9f05c792748af4ef0c04b414c324ed87 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoi Date: Tue, 18 Mar 2025 14:34:59 +0200 Subject: [PATCH 28/31] Create pg_tde dir on replicas (#149) Otherwise `pg_tde_add_key_provider...` fill fail if a replica was created before primary run `CREATE EXTENSION pg_tde` Fixes PG-1489 --- contrib/pg_tde/src/pg_tde.c | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/pg_tde/src/pg_tde.c b/contrib/pg_tde/src/pg_tde.c index d89261e9328..c4cddbe5828 100644 --- a/contrib/pg_tde/src/pg_tde.c +++ b/contrib/pg_tde/src/pg_tde.c @@ -171,6 +171,7 @@ pg_tde_extension_initialize(PG_FUNCTION_ARGS) void extension_install_redo(XLogExtensionInstall *xlrec) { + pg_tde_init_data_dir(); run_extension_install_callbacks(xlrec, true); } From 01a8ee3e03c5dc2750c6793b57344b9d44a83b12 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoi Date: Tue, 18 Mar 2025 23:41:38 +0200 Subject: [PATCH 29/31] Update WAL keys pointers after resizing the key cache (#151) After the tde_rel_key_cache resize, pointers in tde_wal_key_cache became invalid. So we have to update those pointers after the resize. For PG-1488 --- contrib/pg_tde/src/access/pg_tde_tdemap.c | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/contrib/pg_tde/src/access/pg_tde_tdemap.c b/contrib/pg_tde/src/access/pg_tde_tdemap.c index 4d0dc5ed236..4eba8eab58a 100644 --- a/contrib/pg_tde/src/access/pg_tde_tdemap.c +++ b/contrib/pg_tde/src/access/pg_tde_tdemap.c @@ -120,6 +120,9 @@ RelKeyCache tde_rel_key_cache = { }; +/* + * TODO: WAL should have its own RelKeyCache + */ static WALKeyCacheRec *tde_wal_key_cache = NULL; static WALKeyCacheRec *tde_wal_key_last_rec = NULL; @@ -147,6 +150,7 @@ static void pg_tde_write_keydata(char *db_keydata_path, TDEPrincipalKeyInfo *pri static void pg_tde_write_one_keydata(int keydata_fd, int32 key_index, InternalKey *enc_rel_key_data); static int keyrotation_init_file(TDEPrincipalKeyInfo *new_principal_key_info, char *rotated_filename, char *filename, bool *is_new_file, off_t *curr_pos); static void finalize_key_rotation(char *m_path_old, char *k_path_old, char *m_path_new, char *k_path_new); +static void update_wal_keys_cache(void); InternalKey * pg_tde_create_smgr_key(const RelFileLocatorBackend *newrlocator) @@ -1552,6 +1556,25 @@ pg_tde_get_wal_cache_keys(void) return tde_wal_key_cache; } +/* Updates WAL keys cache pointers */ +static void +update_wal_keys_cache(void) +{ + WALKeyCacheRec *wal_rec = tde_wal_key_cache; + RelFileLocator rlocator = GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); + + for (int i = 0; i < tde_rel_key_cache.len && wal_rec; i++) + { + RelKeyCacheRec *rec = tde_rel_key_cache.data + i; + + if (RelFileLocatorEquals(rec->locator, rlocator)) + { + wal_rec->key = &rec->key; + wal_rec = wal_rec->next; + } + } +} + InternalKey * pg_tde_read_last_wal_key(void) { @@ -1789,6 +1812,9 @@ pg_tde_put_key_into_cache(const RelFileLocator *rlocator, InternalKey *key) elog(WARNING, "could not mlock internal key cache pages: %m"); tde_rel_key_cache.cap = (size - 1) / sizeof(RelKeyCacheRec); + + /* update wal key pointers after moving the cache */ + update_wal_keys_cache(); } rec = tde_rel_key_cache.data + tde_rel_key_cache.len; From 3bf4f5c45d570aefb7e48b1f382dfa679eb002b2 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Tue, 18 Mar 2025 22:32:15 +0000 Subject: [PATCH 30/31] PG-1488: Only create keys for main/init forks smgr_create is called for all forks. It is possible that additional forks for existing tables are created during a tde creation event. In practice this happens quite often with CREATE INDEX CONCURRENTLY. Without this fix, pg_tde created encryption keys for these existing tables, and later writes and reads tried to use these keys with all these issues. In practice some of the file got encrypted, some didn't, and earlier records that weren't encrypted became unreadable. --- contrib/pg_tde/src/smgr/pg_tde_smgr.c | 36 +++++++++++++++++---------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/contrib/pg_tde/src/smgr/pg_tde_smgr.c b/contrib/pg_tde/src/smgr/pg_tde_smgr.c index 6104b9e49b3..fe7e9140dd0 100644 --- a/contrib/pg_tde/src/smgr/pg_tde_smgr.c +++ b/contrib/pg_tde/src/smgr/pg_tde_smgr.c @@ -242,20 +242,30 @@ tde_mdcreate(RelFileLocator relold, SMgrRelation reln, ForkNumber forknum, bool mdcreate(relold, reln, forknum, isRedo); - /* - * Later calls then decide to encrypt or not based on the existence of the - * key - */ - key = tde_smgr_get_key(reln, event->alterAccessMethodMode ? NULL : &relold, true); - - if (key) - { - tdereln->encrypted_relation = true; - tdereln->relKey = *key; - } - else + if (forknum == MAIN_FORKNUM || forknum == INIT_FORKNUM) { - tdereln->encrypted_relation = false; + /* + * Only create keys when creating the main/init fork. Other forks can + * be created later, even during tde creation events. We definitely do + * not want to create keys then, even later, when we encrypt all + * forks! + */ + + /* + * Later calls then decide to encrypt or not based on the existence of + * the key + */ + key = tde_smgr_get_key(reln, event->alterAccessMethodMode ? NULL : &relold, true); + + if (key) + { + tdereln->encrypted_relation = true; + tdereln->relKey = *key; + } + else + { + tdereln->encrypted_relation = false; + } } } From e92fee7662aa78ea7912811abda378d4dda76ea0 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoi Date: Fri, 21 Mar 2025 21:28:52 +0200 Subject: [PATCH 31/31] Pass proper old rlocator to smgr when creating relations and indexes (#156) --- contrib/pg_tde/Makefile | 3 +- contrib/pg_tde/expected/recreate_storage.out | 134 +++++++++++++++++++ contrib/pg_tde/meson.build | 1 + contrib/pg_tde/sql/recreate_storage.sql | 54 ++++++++ src/backend/bootstrap/bootparse.y | 3 +- src/backend/catalog/heap.c | 38 +++++- src/backend/catalog/index.c | 9 +- src/backend/catalog/toasting.c | 2 +- src/backend/commands/indexcmds.c | 2 +- src/include/catalog/heap.h | 4 +- src/include/catalog/index.h | 4 +- 11 files changed, 241 insertions(+), 13 deletions(-) create mode 100644 contrib/pg_tde/expected/recreate_storage.out create mode 100644 contrib/pg_tde/sql/recreate_storage.sql diff --git a/contrib/pg_tde/Makefile b/contrib/pg_tde/Makefile index 50c839e5b9a..c3d49c93c85 100644 --- a/contrib/pg_tde/Makefile +++ b/contrib/pg_tde/Makefile @@ -25,7 +25,8 @@ alter_index_basic \ merge_join_basic \ tablespace_basic \ relocate \ -version +version \ +recreate_storage TAP_TESTS = 1 OBJS = src/encryption/enc_tde.o \ diff --git a/contrib/pg_tde/expected/recreate_storage.out b/contrib/pg_tde/expected/recreate_storage.out new file mode 100644 index 00000000000..bc361d73dc6 --- /dev/null +++ b/contrib/pg_tde/expected/recreate_storage.out @@ -0,0 +1,134 @@ +CREATE EXTENSION IF NOT EXISTS pg_tde; +SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_key_provider_file +------------------------------ + 1 +(1 row) + +SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); + pg_tde_set_principal_key +-------------------------- + t +(1 row) + +SET default_table_access_method = "tde_heap"; +CREATE TABLE t1(n integer); +SELECT pg_tde_is_encrypted('t1'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +VACUUM FULL t1; +SELECT pg_tde_is_encrypted('t1'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +CREATE TABLE test_tab1 AS SELECT generate_series(1,10) a; +CREATE INDEX test_idx1 ON test_tab1(a); +SELECT pg_tde_is_encrypted('test_tab1'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +SELECT pg_tde_is_encrypted('test_idx1'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +REINDEX index CONCURRENTLY test_idx1; +SELECT pg_tde_is_encrypted('test_tab1'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +SELECT pg_tde_is_encrypted('test_idx1'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +CREATE TABLE mvtest_t (id int NOT NULL PRIMARY KEY, type text NOT NULL, amt numeric NOT NULL); +INSERT INTO mvtest_t VALUES + (1, 'x', 2), + (2, 'x', 3), + (3, 'y', 5), + (4, 'y', 7), + (5, 'z', 11); +CREATE MATERIALIZED VIEW mvtest_tm AS SELECT type, sum(amt) AS totamt FROM mvtest_t GROUP BY type WITH NO DATA; +SELECT pg_tde_is_encrypted('mvtest_tm'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +REFRESH MATERIALIZED VIEW mvtest_tm; +SELECT pg_tde_is_encrypted('mvtest_tm'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +CREATE TYPE rewritetype AS (a int); +CREATE TABLE rewritemetoo1 OF rewritetype; +CREATE TABLE rewritemetoo2 OF rewritetype; +SELECT pg_tde_is_encrypted('rewritemetoo1'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +SELECT pg_tde_is_encrypted('rewritemetoo2'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +ALTER TYPE rewritetype ALTER ATTRIBUTE a TYPE text cascade; +SELECT pg_tde_is_encrypted('rewritemetoo1'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +SELECT pg_tde_is_encrypted('rewritemetoo2'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +CREATE TABLE encrypted_table ( + id SERIAL, + data TEXT, + created_at DATE NOT NULL, + PRIMARY KEY (id, created_at) +) USING tde_heap; +CREATE INDEX idx_date ON encrypted_table (created_at); +SELECT pg_tde_is_encrypted('encrypted_table'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +CLUSTER encrypted_table USING idx_date; +SELECT pg_tde_is_encrypted('encrypted_table'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +DROP EXTENSION pg_tde CASCADE; +NOTICE: drop cascades to 7 other objects +DETAIL: drop cascades to table t1 +drop cascades to table test_tab1 +drop cascades to table mvtest_t +drop cascades to materialized view mvtest_tm +drop cascades to table rewritemetoo1 +drop cascades to table rewritemetoo2 +drop cascades to table encrypted_table +RESET default_table_access_method; diff --git a/contrib/pg_tde/meson.build b/contrib/pg_tde/meson.build index 19d55ef5e1d..4316828dc29 100644 --- a/contrib/pg_tde/meson.build +++ b/contrib/pg_tde/meson.build @@ -108,6 +108,7 @@ sql_tests = [ 'update_basic', 'key_provider', 'relocate', + 'recreate_storage', ] tap_tests = [ diff --git a/contrib/pg_tde/sql/recreate_storage.sql b/contrib/pg_tde/sql/recreate_storage.sql new file mode 100644 index 00000000000..47be8f15e1e --- /dev/null +++ b/contrib/pg_tde/sql/recreate_storage.sql @@ -0,0 +1,54 @@ +CREATE EXTENSION IF NOT EXISTS pg_tde; + +SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); + +SET default_table_access_method = "tde_heap"; + +CREATE TABLE t1(n integer); +SELECT pg_tde_is_encrypted('t1'); +VACUUM FULL t1; +SELECT pg_tde_is_encrypted('t1'); + +CREATE TABLE test_tab1 AS SELECT generate_series(1,10) a; +CREATE INDEX test_idx1 ON test_tab1(a); +SELECT pg_tde_is_encrypted('test_tab1'); +SELECT pg_tde_is_encrypted('test_idx1'); +REINDEX index CONCURRENTLY test_idx1; +SELECT pg_tde_is_encrypted('test_tab1'); +SELECT pg_tde_is_encrypted('test_idx1'); + +CREATE TABLE mvtest_t (id int NOT NULL PRIMARY KEY, type text NOT NULL, amt numeric NOT NULL); +INSERT INTO mvtest_t VALUES + (1, 'x', 2), + (2, 'x', 3), + (3, 'y', 5), + (4, 'y', 7), + (5, 'z', 11); +CREATE MATERIALIZED VIEW mvtest_tm AS SELECT type, sum(amt) AS totamt FROM mvtest_t GROUP BY type WITH NO DATA; +SELECT pg_tde_is_encrypted('mvtest_tm'); +REFRESH MATERIALIZED VIEW mvtest_tm; +SELECT pg_tde_is_encrypted('mvtest_tm'); + +CREATE TYPE rewritetype AS (a int); +CREATE TABLE rewritemetoo1 OF rewritetype; +CREATE TABLE rewritemetoo2 OF rewritetype; +SELECT pg_tde_is_encrypted('rewritemetoo1'); +SELECT pg_tde_is_encrypted('rewritemetoo2'); +ALTER TYPE rewritetype ALTER ATTRIBUTE a TYPE text cascade; +SELECT pg_tde_is_encrypted('rewritemetoo1'); +SELECT pg_tde_is_encrypted('rewritemetoo2'); + +CREATE TABLE encrypted_table ( + id SERIAL, + data TEXT, + created_at DATE NOT NULL, + PRIMARY KEY (id, created_at) +) USING tde_heap; +CREATE INDEX idx_date ON encrypted_table (created_at); +SELECT pg_tde_is_encrypted('encrypted_table'); +CLUSTER encrypted_table USING idx_date; +SELECT pg_tde_is_encrypted('encrypted_table'); + +DROP EXTENSION pg_tde CASCADE; +RESET default_table_access_method; diff --git a/src/backend/bootstrap/bootparse.y b/src/backend/bootstrap/bootparse.y index 3c9c1da0216..34ce18a3660 100644 --- a/src/backend/bootstrap/bootparse.y +++ b/src/backend/bootstrap/bootparse.y @@ -210,7 +210,8 @@ Boot_CreateStmt: true, &relfrozenxid, &relminmxid, - true); + true, + NULL); elog(DEBUG4, "bootstrap relation created"); } else diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 6ebe07e2c00..794bf3dc636 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -300,7 +300,8 @@ heap_create(const char *relname, bool allow_system_table_mods, TransactionId *relfrozenxid, MultiXactId *relminmxid, - bool create_storage) + bool create_storage, + RelFileLocator *old_rlocator) { Relation rel; @@ -385,14 +386,31 @@ heap_create(const char *relname, */ if (create_storage) { + RelFileLocator prev_rlocator = rel->rd_locator; + RelFileLocator new_rlocator = rel->rd_locator; + + if (old_rlocator != NULL) + { + prev_rlocator = *old_rlocator; + + /* + * table_relation_set_new_filelocator() takes old_rlocator from + * rel->rd_locator + */ + rel->rd_locator = prev_rlocator; + } + if (RELKIND_HAS_TABLE_AM(rel->rd_rel->relkind)) - table_relation_set_new_filelocator(rel, &rel->rd_locator, + table_relation_set_new_filelocator(rel, &new_rlocator, relpersistence, relfrozenxid, relminmxid); else if (RELKIND_HAS_STORAGE(rel->rd_rel->relkind)) - RelationCreateStorage(rel->rd_locator, rel->rd_locator, relpersistence, true); + RelationCreateStorage(prev_rlocator, new_rlocator, relpersistence, true); else Assert(false); + + /* restore the orginal rel's locator */ + rel->rd_locator = new_rlocator; } /* @@ -1129,6 +1147,8 @@ heap_create_with_catalog(const char *relname, Oid existing_relid; Oid old_type_oid; Oid new_type_oid; + RelFileLocator *old_rlocator = NULL; + Relation old_rel; /* By default set to InvalidOid unless overridden by binary-upgrade */ RelFileNumber relfilenumber = InvalidRelFileNumber; @@ -1283,6 +1303,12 @@ heap_create_with_catalog(const char *relname, else relacl = NULL; + if (relrewrite != InvalidOid) + { + old_rel = table_open(relrewrite, AccessShareLock); + old_rlocator = &old_rel->rd_locator; + } + /* * Create the relcache entry (mostly dummy at this point) and the physical * disk file. (If we fail further down, it's the smgr's responsibility to @@ -1306,7 +1332,11 @@ heap_create_with_catalog(const char *relname, allow_system_table_mods, &relfrozenxid, &relminmxid, - true); + true, + old_rlocator); + + if (relrewrite != InvalidOid) + table_close(old_rel, AccessShareLock); Assert(relid == RelationGetRelid(new_rel_desc)); diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 416585fcf51..706ab4cfa6a 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -741,7 +741,8 @@ index_create(Relation heapRelation, bits16 constr_flags, bool allow_system_table_mods, bool is_internal, - Oid *constraintId) + Oid *constraintId, + RelFileLocator *old_rlocator) { Oid heapRelationId = RelationGetRelid(heapRelation); Relation pg_class; @@ -985,7 +986,8 @@ index_create(Relation heapRelation, allow_system_table_mods, &relfrozenxid, &relminmxid, - create_storage); + create_storage, + old_rlocator); Assert(relfrozenxid == InvalidTransactionId); Assert(relminmxid == InvalidMultiXactId); @@ -1459,7 +1461,8 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId, 0, true, /* allow table to be a system catalog? */ false, /* is_internal? */ - NULL); + NULL, + &indexRelation->rd_locator); /* Close the relations used and clean up */ index_close(indexRelation, NoLock); diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index ad3082c62ac..fa68de27bca 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -325,7 +325,7 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, BTREE_AM_OID, rel->rd_rel->reltablespace, collationIds, opclassIds, NULL, coloptions, NULL, (Datum) 0, - INDEX_CREATE_IS_PRIMARY, 0, true, true, NULL); + INDEX_CREATE_IS_PRIMARY, 0, true, true, NULL, NULL); table_close(toast_rel, NoLock); diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index f7fe1fae91c..ff96a56c9e9 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -1208,7 +1208,7 @@ DefineIndex(Oid tableId, coloptions, NULL, reloptions, flags, constr_flags, allowSystemTableMods, !check_rights, - &createdConstraintId); + &createdConstraintId, NULL); ObjectAddressSet(address, RelationRelationId, indexRelationId); diff --git a/src/include/catalog/heap.h b/src/include/catalog/heap.h index c512824cd1c..fc5a13f0b83 100644 --- a/src/include/catalog/heap.h +++ b/src/include/catalog/heap.h @@ -17,6 +17,7 @@ #include "catalog/indexing.h" #include "catalog/objectaddress.h" #include "parser/parse_node.h" +#include "storage/relfilelocator.h" /* flag bits for CheckAttributeType/CheckAttributeNamesTypes */ @@ -60,7 +61,8 @@ extern Relation heap_create(const char *relname, bool allow_system_table_mods, TransactionId *relfrozenxid, MultiXactId *relminmxid, - bool create_storage); + bool create_storage, + RelFileLocator *old_rlocator); extern Oid heap_create_with_catalog(const char *relname, Oid relnamespace, diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h index 7d434f8e653..b8b9e67e6db 100644 --- a/src/include/catalog/index.h +++ b/src/include/catalog/index.h @@ -16,6 +16,7 @@ #include "catalog/objectaddress.h" #include "nodes/execnodes.h" +#include "storage/relfilelocator.h" #define DEFAULT_INDEX_TYPE "btree" @@ -86,7 +87,8 @@ extern Oid index_create(Relation heapRelation, bits16 constr_flags, bool allow_system_table_mods, bool is_internal, - Oid *constraintId); + Oid *constraintId, + RelFileLocator *old_rlocator); #define INDEX_CONSTR_CREATE_MARK_AS_PRIMARY (1 << 0) #define INDEX_CONSTR_CREATE_DEFERRABLE (1 << 1)