Merge tag 'release-17.4.1' into TDE_REL_17_STABLE

Conflicts

- contrib/pg_tde/Makefile
- contrib/pg_tde/expected/change_access_method.out
- contrib/pg_tde/meson.build
- contrib/pg_tde/src/access/pg_tde_tdemap.c
- contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c
- contrib/pg_tde/src/keyring/keyring_vault.c
- contrib/pg_tde/src/pg_tde.c
- contrib/pg_tde/src/pg_tde_alter_key_provider.c
- contrib/pg_tde/src/pg_tde_event_capture.c
- src/bin/Makefile
- src/bin/meson.build

Deleted in TDE_REL_17_STABLE

- contrib/pg_tde/expected/change_access_method_basic.out
- contrib/pg_tde/expected/vault_v2_test_basic.out
- contrib/pg_tde/sql/change_access_method.inc
- contrib/pg_tde/sql/vault_v2_test.inc
- src/bin/pg_tde_change_key_provider/Makefile
- src/bin/pg_tde_change_key_provider/meson.build
pull/209/head
Andreas Karlsson 6 months ago
commit 3a6422b9f9
  1. 7
      contrib/pg_tde/Makefile
  2. 34
      contrib/pg_tde/documentation/docs/apt.md
  3. 2
      contrib/pg_tde/documentation/docs/contribute.md
  4. 4
      contrib/pg_tde/documentation/docs/faq.md
  5. 24
      contrib/pg_tde/documentation/docs/functions.md
  6. 68
      contrib/pg_tde/documentation/docs/install.md
  7. 2
      contrib/pg_tde/documentation/docs/multi-tenant-setup.md
  8. 2
      contrib/pg_tde/documentation/docs/release-notes/release-notes.md
  9. 4
      contrib/pg_tde/documentation/docs/setup.md
  10. 5
      contrib/pg_tde/documentation/docs/switch.md
  11. 8
      contrib/pg_tde/documentation/docs/tde.md
  12. 23
      contrib/pg_tde/documentation/docs/yum.md
  13. 24
      contrib/pg_tde/expected/change_access_method.out
  14. 3
      contrib/pg_tde/expected/default_principal_key.out
  15. 3
      contrib/pg_tde/expected/default_principal_key_1.out
  16. 74
      contrib/pg_tde/expected/delete_key_provider.out
  17. 3
      contrib/pg_tde/expected/key_provider.out
  18. 3
      contrib/pg_tde/expected/key_provider_1.out
  19. 8
      contrib/pg_tde/expected/no_provider_error.out
  20. 134
      contrib/pg_tde/expected/recreate_storage.out
  21. 8
      contrib/pg_tde/expected/version.out
  22. 11
      contrib/pg_tde/meson.build
  23. 6
      contrib/pg_tde/pg_tde--1.0-rc.sql
  24. 8
      contrib/pg_tde/sql/change_access_method.sql
  25. 20
      contrib/pg_tde/sql/delete_key_provider.sql
  26. 11
      contrib/pg_tde/sql/no_provider_error.sql
  27. 54
      contrib/pg_tde/sql/recreate_storage.sql
  28. 3
      contrib/pg_tde/sql/version.sql
  29. 31
      contrib/pg_tde/src/access/pg_tde_tdemap.c
  30. 26
      contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c
  31. 3
      contrib/pg_tde/src/catalog/tde_keyring.c
  32. 7
      contrib/pg_tde/src/catalog/tde_principal_key.c
  33. 10
      contrib/pg_tde/src/include/pg_tde_event_capture.h
  34. 2
      contrib/pg_tde/src/keyring/keyring_kmip.c
  35. 2
      contrib/pg_tde/src/keyring/keyring_vault.c
  36. 10
      contrib/pg_tde/src/pg_tde.c
  37. 51
      contrib/pg_tde/src/pg_tde_change_key_provider.c
  38. 83
      contrib/pg_tde/src/pg_tde_event_capture.c
  39. 44
      contrib/pg_tde/src/smgr/pg_tde_smgr.c
  40. 3
      src/backend/bootstrap/bootparse.y
  41. 38
      src/backend/catalog/heap.c
  42. 9
      src/backend/catalog/index.c
  43. 2
      src/backend/catalog/toasting.c
  44. 2
      src/backend/commands/indexcmds.c
  45. 1
      src/bin/pg_tde_change_key_provider/.gitignore
  46. 4
      src/include/catalog/heap.h
  47. 4
      src/include/catalog/index.h

@ -18,6 +18,7 @@ key_provider \
keyprovider_dependency \
kmip_test \
pg_tde_is_encrypted \
recreate_storage \
relocate \
subtransaction \
tablespace \
@ -52,9 +53,9 @@ src/libkmip/libkmip/src/kmip_bio.o \
src/libkmip/libkmip/src/kmip_locate.o \
src/libkmip/libkmip/src/kmip_memset.o
SCRIPTS_built = src/pg_tde_alter_key_provider
SCRIPTS_built = src/pg_tde_change_key_provider
EXTRA_CLEAN += src/pg_tde_alter_key_provider.o
EXTRA_CLEAN += src/pg_tde_change_key_provider.o
ifdef USE_PGXS
PG_CONFIG = pg_config
@ -71,7 +72,7 @@ endif
override SHLIB_LINK += -lcurl -lcrypto -lssl
src/pg_tde_alter_key_provider: src/pg_tde_alter_key_provider.o $(top_srcdir)/src/fe_utils/simple_list.o $(top_builddir)/src/libtde/libtde.a
src/pg_tde_change_key_provider: src/pg_tde_change_key_provider.o $(top_srcdir)/src/fe_utils/simple_list.o $(top_builddir)/src/libtde/libtde.a
$(CC) -DFRONTEND $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
# Fetches typedefs list for PostgreSQL core and merges it with typedefs defined in this project.

@ -10,7 +10,7 @@ Check the [list of supported platforms](install.md#__tabbed_1_1).
2. You need the `percona-release` repository management tool that enables the desired Percona repository for you.
### Install `percona-release`
### Install `percona-release` {.power-number}
1. You need the following dependencies to install `percona-release`:
@ -21,26 +21,26 @@ Check the [list of supported platforms](install.md#__tabbed_1_1).
Install them with the following command:
```bash
sudo apt-get install -y wget gnupg2 curl lsb-release
```{.bash data-prompt="$"}
$ sudo apt-get install -y wget gnupg2 curl lsb-release
```
2. Fetch the `percona-release` package
```bash
sudo wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb
```{.bash data-prompt="$"}
$ sudo wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb
```
3. Install `percona-release`
```bash
sudo dpkg -i percona-release_latest.generic_all.deb
```{.bash data-prompt="$"}
$ sudo dpkg -i percona-release_latest.generic_all.deb
```
4. Enable the Percona Distribution for PostgreSQL repository
```{.bash data-prompt="$"}
$ sudo percona-release enable ppg-{{pgversion17}}
$ sudo percona-release enable-only ppg-{{pgversion17}}
```
6. Update the local cache
@ -49,30 +49,30 @@ Check the [list of supported platforms](install.md#__tabbed_1_1).
sudo apt-get update
```
## Install `pg_tde`
## Install `pg_tde` {.power-number}
!!! important
The `pg_tde` {{release}} extension is a part of the `percona-postgresql-17` package. If you installed a previous version of `pg_tde` from the `percona-postgresql-17-pg-tde` package, do the following:
The `pg_tde` extension is a part of the `percona-postgresql-17` package. If you installed a previous version of `pg_tde` from the `percona-postgresql-17-pg-tde` package, do the following:
1. Drop the extension using the `DROP EXTENSION` with `CASCADE` command.
The use of the `CASCADE` parameter deletes all tables that were created in the database with `pg_tde` enabled and also all dependencies upon the encrypted table (e.g. foreign keys in a non-encrypted table used in the encrypted one).
The use of the `CASCADE` parameter deletes all tables that were created in the database with `pg_tde` enabled and also all dependencies upon the encrypted table (e.g. foreign keys in a non-encrypted table used in the encrypted one).
```sql
DROP EXTENSION pg_tde CASCADE
```
```sql
DROP EXTENSION pg_tde CASCADE
```
2. Uninstall the `percona-postgresql-17-pg-tde` package.
After all [preconditions](#preconditions) are met, run the following command to install `pg_tde`:
```bash
sudo apt-get install -y percona-postgresql-17
```{.bash data-prompt="$"}
$ sudo apt-get install -y percona-postgresql-17
```
## Next step
[Setup](setup.md){.md-button}
[Setup :material-arrow-right:](setup.md){.md-button}

@ -64,7 +64,7 @@ You can run tests on your local machine with whatever operating system you have.
## Contribute to documentation
`pg_tde` documentation is written in Markdown language, so you can [write the docs for your code changes](#write-the-docs) or
[edit the existing documentation online via GitHub](#edit-documentation-online-vi-github). If you wish to have more control over the doc process, jump to how to [edit documentation locally](#edit-documentation-locally).
[edit the existing documentation online via GitHub](#edit-documentation-online-via-github). If you wish to have more control over the doc process, jump to how to [edit documentation locally](#edit-documentation-locally).
Before you start, learn what [Markdown] is and how to write it. For your convenience, there's also a [Markdown cheat sheet] to help you with the syntax.

@ -98,9 +98,9 @@ Whenever the WAL is being read (by the recovery process or tools), the decision
It depends on your business requirements and the sensitivity of your data. Encrypting all data is a good practice but it can have a performance impact.
Consider encrypting only tables that store sensitive data. You can decide what tables to encrypt and with what key. The [Setup](setup.md) section in documentation focuses on this approach.
Consider encrypting only tables that store sensitive data. You can decide what tables to encrypt and with what key. The [Set up multi-tenancy](multi-tenant-setup.md) section in the documentation focuses on this approach.
We advise encrypting the whole database only if all your data is sensitive, like PII, or if there is no other way to comply with data safety requirements. See [How to configure global encryption](global-encryption.md).
We advise encrypting the whole database only if all your data is sensitive, like PII, or if there is no other way to comply with data safety requirements.
## What cipher mechanisms are used by `pg_tde`?

@ -36,8 +36,8 @@ These functions allow or revoke the use of the permissions management functions:
Use these functions to grant or revoke the use of query functions, which do not modify the encryption settings:
* `pg_tde_grant_key_viewer_management_to_role(role)`
* `pg_tde_revoke_key_viewer_management_from_role(role)`
* `pg_tde_grant_key_viewer_to_role(role)`
* `pg_tde_revoke_key_viewer_from_role(role)`
## Key provider management
@ -203,7 +203,7 @@ Princial keys are stored on key providers by the name specified in this function
### pg_tde_set_principal_key
Creates or rotates the principal key for the current database using the specified key provider and name.
Creates or rotates the principal key for the current database using the specified database key provider and key name.
```
SELECT pg_tde_set_principal_key('name-of-the-principal-key','provider-name','ensure_new_key');
@ -215,6 +215,20 @@ SELECT pg_tde_set_principal_key('name-of-the-principal-key','provider-name','ens
If the provider already stores a key by that name, the function returns an error.
* If set to `false`, an existing principal key may be reused.
### pg_tde_set_global_principal_key
Creates or rotates the principal key for the current database using the specified global key provider and key name.
```
SELECT pg_tde_set_global_principal_key('name-of-the-principal-key','provider-name','ensure_new_key');
```
The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation:
* If set to `true` (default), a new key must be unique.
If the provider already stores a key by that name, the function returns an error.
* If set to `false`, an existing principal key may be reused.
### pg_tde_set_server_principal_key
Creates or rotates the global principal key using the specified key provider. Use this function to set a principal key for WAL encryption.
@ -234,7 +248,7 @@ The `ensure_new_key` parameter instructs the function how to handle a principal
Creates or rotates the default principal key for the server using the specified key provider.
The default key is automatically used as a principal key by any database that doesn't have a specific principal key created the first time an encrypted database object is created.
The default key is automatically used as a principal key by any database that doesn't have a specific principal key already created when the first encrypted database object is created.
```
SELECT pg_tde_set_default_principal_key('name-of-the-principal-key','provider-name','ensure_new_key');
@ -264,7 +278,7 @@ You can also verify if the table in a custom schema is encrypted. Pass the schem
SELECT pg_tde_is_encrypted('schema.table_name');
```
This can additoonally be used the verify that indexes and sequences are encrypted.
This can additionally be used to verify that indexes and sequences are encrypted.
### pg_tde_principal_key_info

@ -1,9 +1,8 @@
# Installation
Install `pg_tde` using one of available installation methods:
Install `pg_tde` using one of the available installation methods:
=== "Package manager"
=== ":octicons-terminal-16: Package manager"
The packages are available for the following operating systems:
@ -15,67 +14,22 @@ Install `pg_tde` using one of available installation methods:
- Debian 11 (Bullseye)
- Debian 12 (Bookworm)
[Install on Debian or Ubuntu](apt.md){.md-button}
[Install on RHEL or derivatives](yum.md){.md-button}
=== "Build from source"
To build `pg_tde` from source code, do the following:
1. On Ubuntu/Debian: Install the following dependencies required for the build:
```sh
sudo apt install make gcc postgresql-server-dev-17 libcurl4-openssl-dev
```
2. [Install Percona Distribution for PostgreSQL 17 :octicons-link-external-16:](https://docs.percona.com/postgresql/17/installing.html) or [upstream PostgreSQL 17 :octicons-link-external-16:](https://www.postgresql.org/download/)
3. If PostgreSQL is installed in a non standard directory, set the `PG_CONFIG` environment variable to point to the `pg_config` executable.
4. Clone the repository:
```
git clone git://github.com/percona/pg_tde
```
5. Compile and install the extension
```
cd pg_tde
make USE_PGXS=1
sudo make USE_PGXS=1 install
```
=== "Run in Docker"
!!! note
The steps below are for the deprecated PostgreSQL Community version of `pg_tde` and apply only to PostgreSQL 16. This `pg_tde` version provides only the `tde_heap_basic` access method for data encryption.
To enjoy full encryption features, run the `pg_tde` version for Percona Server for PostgreSQL. [Use the Percona Distribution for PostgreSQL Docker image :octicons-link-external-16:](https://docs.percona.com/postgresql/17/docker.html).
You can find Docker images on [Docker Hub](https://hub.docker.com/r/perconalab/pg_tde). Images are built on top of [postgres:16](https://hub.docker.com/_/postgres) official image.
To run `pg_tde` in Docker, use the following command:
[Install on Debian or Ubuntu :material-arrow-right:](apt.md){.md-button}
[Install on RHEL or derivatives :material-arrow-right:](yum.md){.md-button}
```
docker run --name pg-tde -e POSTGRES_PASSWORD=mysecretpassword -d perconalab/pg_tde
```
=== ":simple-docker: Docker"
It builds and adds the `pg_tde` extension to PostgreSQL 16. The `postgresql.conf` contains the required modifications. The `pg_tde` extension is added to `template1` so that all new databases automatically have the `pg_tde` extension loaded.
`pg_tde` is a part of the Percona Distribution for PostgreSQL Docker image. Use this image to enjoy full encryption capabilities. Check below to get access to a detailed step-by-step guide.
Keys are not created automatically. You must configure a key provider and a principal key for each database where you wish to use encrypted tables.
[Run in Docker :material-arrow-right:](https://docs.percona.com/postgresql/latest/docker.html)
Connect to the container and establish the `psql` session there. Then, see the instructions in the [Setup](setup.md) section, starting with the 4th point, as the first 3 steps are already completed in the Docker image.
=== ":octicons-download-16: Manual download"
See [Docker Docs](https://hub.docker.com/_/postgres) on usage.
`pg_tde` is included in the Percona Distribution for PostgreSQL tarball. Check below to get access to a detailed step-by-step guide.
You can also build a Docker image manually with:
[Install from tarballs](https://docs.percona.com/postgresql/17/tarball.html) guide for instructions.
```
docker build . -f ./docker/Dockerfile -t your-image-name
```
## Next steps
[Setup](setup.md){.md-button}
[Setup :material-arrow-right:](setup.md){.md-button}

@ -4,7 +4,7 @@ The steps below describe how to set up multi-tenancy with `pg_tde`. Multi-tenanc
If you don't need multi-tenancy, use the global key provider. See the configuration steps from the [Setup](setup.md) section.
For how to enable WAL encryption, refer to the [WAL encryption](setup.md#wal-encryption) section.
For how to enable WAL encryption, refer to the [WAL encryption](wal-encryption.md) section.
--8<-- "kms-considerations.md"

@ -1,6 +1,6 @@
# pg_tde release notes
`pg_tde` extension brings in [Transparent Data Encryption (TDE)](tde.md) to PostgreSQL and enables you to keep sensitive data safe and secure.
`pg_tde` extension brings in [Transparent Data Encryption (TDE)](../tde.md) to PostgreSQL and enables you to keep sensitive data safe and secure.
[Get started](../install.md){.md-button}

@ -19,7 +19,7 @@ Load the `pg_tde` at startup time. The extension requires additional shared memo
* On Debian and Ubuntu:
```sh
sudo systemctl restart postgresql-17
sudo systemctl restart postgresql@17-main
```
* On RHEL and derivatives
@ -64,6 +64,8 @@ Load the `pg_tde` at startup time. The extension requires additional shared memo
* `server-certificate` is the path to the certificate file for the KMIP server.
* `client key` is the path to the client key.
<i warning>:material-information: Warning:</i> Note that pg_tde_add_global_key_provider_kmip currently accepts only a combined client key + client certificate for the last parameter of this function named as `client key`.
<i warning>:material-information: Warning:</i> This example is for testing purposes only:
```

@ -1,5 +0,0 @@
# Switch from Percona Server for PostgreSQL to PostgreSQL Community
Percona Server for PostgreSQL and PostgreSQL Community are binary compatible and enable you to switch from one to another. Here's how:
1. If you used the `tde_heap` (tech preview feature) access method for encryption, either re-encrypt the data using the `tde_heap_basic` access method, or [decrypt](decrypt.md) it completely

@ -7,7 +7,7 @@ Transparent Data Encryption is a technology to protect data at rest. The encrypt
To encrypt the data, two types of keys are used:
* Internal encryption keys to encrypt user data. They are stored internally, near the data that they encrypt.
* The principal key to encrypt database keys. It is kept separately from the database keys and is managed externally in the key management store.
* The principal key to encrypt database keys. It is kept separately from the database keys and is managed externally in the key management store.
You have the following options to store and manage principal keys externally:
@ -18,11 +18,11 @@ The encryption process is the following:
![image](_images/tde-flow.png)
When a user creates an encrypted table using `pg_tde`, a new random key is generated for that table using the AES128 (AES-ECB) cipher algorithm. This key is used to encrypt all data the user inserts in that table. Eventually the encrypted data gets stored in the underlying storage.
When a user creates an encrypted table using `pg_tde`, a new random key is generated for that table using the AES128 (AES-ECB) cipher algorithm. This key is used to encrypt all data the user inserts in that table. Eventually the encrypted data gets stored in the underlying storage.
The table itself is encrypted using the principal key. The principal key is stored externally in the key management store.
The table key itself is encrypted using the principal key. The principal key is stored externally in the key management store.
Similarly when the user queries the encrypted table, the principal key is retrieved from the key store to decrypt the table. Then the same unique internal key for that table is used to decrypt the data, and unencrypted data gets returned to the user. So, effectively, every TDE table has a unique key, and each table key is encrypted using the principal key.
Similarly when the user queries the encrypted table, the principal key is retrieved from the key store to decrypt the table key. Then the same unique internal key for that table is used to decrypt the data, and unencrypted data gets returned to the user. So, effectively, every TDE table has a unique key, and each table key is encrypted using the principal key.
## Why do you need TDE?

@ -4,29 +4,27 @@ This tutorial shows how to install `pg_tde` with [Percona Distribution for Postg
Check the [list of supported platforms](install.md#__tabbed_1_1).
## Install `percona-release`
## Install `percona-release` {.power-number}
You need the `percona-release` repository management tool that enables the desired Percona repository for you.
1. Install `percona-release`:
```bash
sudo yum -y install https://repo.percona.com/yum/percona-release-latest.noarch.rpm
```{.bash data-prompt="$"}
$ sudo yum -y install https://repo.percona.com/yum/percona-release-latest.noarch.rpm
```
2. Enable the repository.
Percona provides [two repositories](repo-overview.md) for Percona Distribution for PostgreSQL. We recommend enabling the Major release repository to timely receive the latest updates.
```bash
sudo percona-release enable-only ppg-{{pgversion17}}
```{.bash data-prompt="$"}
$ sudo percona-release enable-only ppg-{{pgversion17}}
```
## Install `pg_tde`
## Install `pg_tde` {.power-number}
!!! important
The `pg_tde` {{release}} extension is a part of the `percona-postgresql17` package. If you installed a previous version of `pg_tde` from the `percona-pg_tde_17` package, do the following:
The `pg_tde` extension is a part of the `percona-postgresql17` package. If you installed a previous version of `pg_tde` from the `percona-pg_tde_17` package, do the following:
1. Drop the extension using the `DROP EXTENSION` with `CASCADE` command.
@ -38,11 +36,12 @@ You need the `percona-release` repository management tool that enables the desir
2. Uninstall the `percona-pg_tde_17` package.
Run the following command to install `pg_tde`:
```bash
sudo yum -y install percona-postgresql17
```{.bash data-prompt="$"}
$ sudo yum -y install percona-postgresql17
```
## Next steps
[Setup](setup.md){.md-button}
[Setup :material-arrow-right:](setup.md){.md-button}

@ -41,6 +41,12 @@ SELECT pg_tde_is_encrypted('country_table_country_id_seq');
t
(1 row)
SELECT pg_tde_is_encrypted('country_table_pkey');
pg_tde_is_encrypted
---------------------
t
(1 row)
-- Try changing the encrypted table to an unencrypted table
ALTER TABLE country_table SET access method heap;
SELECT pg_tde_is_encrypted('country_table_country_id_seq');
@ -49,6 +55,12 @@ SELECT pg_tde_is_encrypted('country_table_country_id_seq');
f
(1 row)
SELECT pg_tde_is_encrypted('country_table_pkey');
pg_tde_is_encrypted
---------------------
f
(1 row)
-- Insert some more data
INSERT INTO country_table (country_name, continent)
VALUES ('France', 'Europe'),
@ -77,6 +89,12 @@ SELECT pg_tde_is_encrypted('country_table_country_id_seq');
f
(1 row)
SELECT pg_tde_is_encrypted('country_table_pkey');
pg_tde_is_encrypted
---------------------
f
(1 row)
-- Change it back to encrypted
ALTER TABLE country_table SET access method tde_heap;
INSERT INTO country_table (country_name, continent)
@ -109,6 +127,12 @@ SELECT pg_tde_is_encrypted('country_table_country_id_seq');
t
(1 row)
SELECT pg_tde_is_encrypted('country_table_pkey');
pg_tde_is_encrypted
---------------------
t
(1 row)
ALTER TABLE country_table ADD y text;
SELECT pg_tde_is_encrypted(('pg_toast.pg_toast_' || 'country_table'::regclass::oid)::regclass);
pg_tde_is_encrypted

@ -17,9 +17,8 @@ ERROR: Can't delete a provider which is currently in use
SELECT id, provider_name FROM pg_tde_list_all_global_key_providers();
id | provider_name
----+---------------
-2 | file-keyring2
-3 | file-provider
(2 rows)
(1 row)
-- Should fail: no principal key for the database yet
SELECT key_provider_id, key_provider_name, principal_key_name

@ -18,9 +18,8 @@ SELECT id, provider_name FROM pg_tde_list_all_global_key_providers();
id | provider_name
----+-----------------
-1 | reg_file-global
-3 | file-keyring2
-4 | file-provider
(3 rows)
(2 rows)
-- Should fail: no principal key for the database yet
SELECT key_provider_id, key_provider_name, principal_key_name

@ -0,0 +1,74 @@
CREATE EXTENSION IF NOT EXISTS pg_tde;
SELECT * FROM pg_tde_principal_key_info();
ERROR: Principal key does not exists for the database
HINT: Use set_principal_key interface to set the principal key
SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per');
pg_tde_add_key_provider_file
------------------------------
1
(1 row)
SELECT * FROM pg_tde_list_all_key_providers();
id | provider_name | provider_type | options
----+---------------+---------------+------------------------------------------------------------
1 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"}
(1 row)
SELECT pg_tde_delete_key_provider('file-provider');
pg_tde_delete_key_provider
----------------------------
(1 row)
SELECT * FROM pg_tde_list_all_key_providers();
id | provider_name | provider_type | options
----+---------------+---------------+---------
(0 rows)
SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per');
pg_tde_add_key_provider_file
------------------------------
2
(1 row)
SELECT * FROM pg_tde_list_all_key_providers();
id | provider_name | provider_type | options
----+---------------+---------------+------------------------------------------------------------
2 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"}
(1 row)
SELECT pg_tde_delete_key_provider('file-provider');
pg_tde_delete_key_provider
----------------------------
(1 row)
SELECT * FROM pg_tde_list_all_key_providers();
id | provider_name | provider_type | options
----+---------------+---------------+---------
(0 rows)
SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per');
pg_tde_add_key_provider_file
------------------------------
3
(1 row)
SELECT * FROM pg_tde_list_all_key_providers();
id | provider_name | provider_type | options
----+---------------+---------------+------------------------------------------------------------
3 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"}
(1 row)
SELECT pg_tde_delete_key_provider('file-provider');
pg_tde_delete_key_provider
----------------------------
(1 row)
SELECT * FROM pg_tde_list_all_key_providers();
id | provider_name | provider_type | options
----+---------------+---------------+---------
(0 rows)
DROP EXTENSION pg_tde;

@ -161,7 +161,6 @@ SELECT pg_tde_delete_global_key_provider('file-keyring2');
SELECT id, provider_name FROM pg_tde_list_all_global_key_providers();
id | provider_name
----+---------------
-2 | file-keyring2
(1 row)
(0 rows)
DROP EXTENSION pg_tde;

@ -165,7 +165,6 @@ SELECT id, provider_name FROM pg_tde_list_all_global_key_providers();
id | provider_name
----+-----------------
-1 | reg_file-global
-3 | file-keyring2
(2 rows)
(1 row)
DROP EXTENSION pg_tde;

@ -0,0 +1,8 @@
CREATE EXTENSION pg_tde;
-- should fail
CREATE TABLE t1 (n INT) USING tde_heap;
ERROR: failed to retrieve principal key. Create one using pg_tde_set_principal_key before using encrypted tables.
-- should work
CREATE TABLE t2 (n INT) USING heap;
DROP TABLE t2;
DROP EXTENSION pg_tde;

@ -0,0 +1,134 @@
CREATE EXTENSION IF NOT EXISTS pg_tde;
SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
pg_tde_add_key_provider_file
------------------------------
1
(1 row)
SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault');
pg_tde_set_principal_key
--------------------------
t
(1 row)
SET default_table_access_method = "tde_heap";
CREATE TABLE t1(n integer);
SELECT pg_tde_is_encrypted('t1');
pg_tde_is_encrypted
---------------------
t
(1 row)
VACUUM FULL t1;
SELECT pg_tde_is_encrypted('t1');
pg_tde_is_encrypted
---------------------
t
(1 row)
CREATE TABLE test_tab1 AS SELECT generate_series(1,10) a;
CREATE INDEX test_idx1 ON test_tab1(a);
SELECT pg_tde_is_encrypted('test_tab1');
pg_tde_is_encrypted
---------------------
t
(1 row)
SELECT pg_tde_is_encrypted('test_idx1');
pg_tde_is_encrypted
---------------------
t
(1 row)
REINDEX index CONCURRENTLY test_idx1;
SELECT pg_tde_is_encrypted('test_tab1');
pg_tde_is_encrypted
---------------------
t
(1 row)
SELECT pg_tde_is_encrypted('test_idx1');
pg_tde_is_encrypted
---------------------
t
(1 row)
CREATE TABLE mvtest_t (id int NOT NULL PRIMARY KEY, type text NOT NULL, amt numeric NOT NULL);
INSERT INTO mvtest_t VALUES
(1, 'x', 2),
(2, 'x', 3),
(3, 'y', 5),
(4, 'y', 7),
(5, 'z', 11);
CREATE MATERIALIZED VIEW mvtest_tm AS SELECT type, sum(amt) AS totamt FROM mvtest_t GROUP BY type WITH NO DATA;
SELECT pg_tde_is_encrypted('mvtest_tm');
pg_tde_is_encrypted
---------------------
t
(1 row)
REFRESH MATERIALIZED VIEW mvtest_tm;
SELECT pg_tde_is_encrypted('mvtest_tm');
pg_tde_is_encrypted
---------------------
t
(1 row)
CREATE TYPE rewritetype AS (a int);
CREATE TABLE rewritemetoo1 OF rewritetype;
CREATE TABLE rewritemetoo2 OF rewritetype;
SELECT pg_tde_is_encrypted('rewritemetoo1');
pg_tde_is_encrypted
---------------------
t
(1 row)
SELECT pg_tde_is_encrypted('rewritemetoo2');
pg_tde_is_encrypted
---------------------
t
(1 row)
ALTER TYPE rewritetype ALTER ATTRIBUTE a TYPE text cascade;
SELECT pg_tde_is_encrypted('rewritemetoo1');
pg_tde_is_encrypted
---------------------
t
(1 row)
SELECT pg_tde_is_encrypted('rewritemetoo2');
pg_tde_is_encrypted
---------------------
t
(1 row)
CREATE TABLE encrypted_table (
id SERIAL,
data TEXT,
created_at DATE NOT NULL,
PRIMARY KEY (id, created_at)
) USING tde_heap;
CREATE INDEX idx_date ON encrypted_table (created_at);
SELECT pg_tde_is_encrypted('encrypted_table');
pg_tde_is_encrypted
---------------------
t
(1 row)
CLUSTER encrypted_table USING idx_date;
SELECT pg_tde_is_encrypted('encrypted_table');
pg_tde_is_encrypted
---------------------
t
(1 row)
DROP EXTENSION pg_tde CASCADE;
NOTICE: drop cascades to 7 other objects
DETAIL: drop cascades to table t1
drop cascades to table test_tab1
drop cascades to table mvtest_t
drop cascades to materialized view mvtest_tm
drop cascades to table rewritemetoo1
drop cascades to table rewritemetoo2
drop cascades to table encrypted_table
RESET default_table_access_method;

@ -0,0 +1,8 @@
CREATE EXTENSION pg_tde;
SELECT pg_tde_version();
pg_tde_version
-----------------
pg_tde 1.0.0-rc
(1 row)
DROP EXTENSION pg_tde;

@ -85,6 +85,7 @@ sql_tests = [
'kmip_test',
'pg_tde_is_encrypted',
'relocate',
'recreate_storage',
'subtransaction',
'tablespace',
'vault_v2_test',
@ -146,16 +147,16 @@ pg_tde_frontend = static_library('pg_tde_frontend',
link_whole: [kmip]
)
pg_tde_alter_key_provider_sources = files(
'src/pg_tde_alter_key_provider.c',
pg_tde_change_key_provider_sources = files(
'src/pg_tde_change_key_provider.c',
)
pg_tde_alter_key_provider = executable('pg_tde_alter_key_provider',
pg_tde_alter_key_provider_sources,
pg_tde_change_key_provider = executable('pg_tde_change_key_provider',
pg_tde_change_key_provider_sources,
dependencies: [frontend_code, lz4, zstd],
c_args: ['-DFRONTEND'], # needed for xlogreader et al
kwargs: default_bin_args,
include_directories: [postgres_inc, pg_tde_inc],
link_with: [pg_tde_frontend]
)
contrib_targets += pg_tde_alter_key_provider
contrib_targets += pg_tde_change_key_provider

@ -581,7 +581,7 @@ BEGIN
EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_set_global_principal_key(text, text, BOOLEAN) TO %I', target_role);
EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_set_server_principal_key(text, text, BOOLEAN) TO %I', target_role);
EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_set_default_principal_key(text, text, BOOLEAN) FROM %I', target_role);
EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_set_default_principal_key(text, text, BOOLEAN) TO %I', target_role);
END;
$$;
@ -628,6 +628,8 @@ BEGIN
EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_principal_key_info() TO %I', target_role);
EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_global_principal_key_info() TO %I', target_role);
EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_verify_principal_key() TO %I', target_role);
EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_verify_global_principal_key() TO %I', target_role);
END;
$$;
@ -708,6 +710,8 @@ BEGIN
EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_principal_key_info() FROM %I', target_role);
EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_global_principal_key_info() FROM %I', target_role);
EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_verify_principal_key() FROM %I', target_role);
EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_verify_global_principal_key() FROM %I', target_role);
END;
$$;

@ -20,11 +20,15 @@ SELECT pg_tde_is_encrypted('country_table');
SELECT pg_tde_is_encrypted('country_table_country_id_seq');
SELECT pg_tde_is_encrypted('country_table_pkey');
-- Try changing the encrypted table to an unencrypted table
ALTER TABLE country_table SET access method heap;
SELECT pg_tde_is_encrypted('country_table_country_id_seq');
SELECT pg_tde_is_encrypted('country_table_pkey');
-- Insert some more data
INSERT INTO country_table (country_name, continent)
VALUES ('France', 'Europe'),
@ -36,6 +40,8 @@ SELECT pg_tde_is_encrypted('country_table');
SELECT pg_tde_is_encrypted('country_table_country_id_seq');
SELECT pg_tde_is_encrypted('country_table_pkey');
-- Change it back to encrypted
ALTER TABLE country_table SET access method tde_heap;
@ -48,6 +54,8 @@ SELECT pg_tde_is_encrypted('country_table');
SELECT pg_tde_is_encrypted('country_table_country_id_seq');
SELECT pg_tde_is_encrypted('country_table_pkey');
ALTER TABLE country_table ADD y text;
SELECT pg_tde_is_encrypted(('pg_toast.pg_toast_' || 'country_table'::regclass::oid)::regclass);

@ -0,0 +1,20 @@
CREATE EXTENSION IF NOT EXISTS pg_tde;
SELECT * FROM pg_tde_principal_key_info();
SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per');
SELECT * FROM pg_tde_list_all_key_providers();
SELECT pg_tde_delete_key_provider('file-provider');
SELECT * FROM pg_tde_list_all_key_providers();
SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per');
SELECT * FROM pg_tde_list_all_key_providers();
SELECT pg_tde_delete_key_provider('file-provider');
SELECT * FROM pg_tde_list_all_key_providers();
SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per');
SELECT * FROM pg_tde_list_all_key_providers();
SELECT pg_tde_delete_key_provider('file-provider');
SELECT * FROM pg_tde_list_all_key_providers();
DROP EXTENSION pg_tde;

@ -0,0 +1,11 @@
CREATE EXTENSION pg_tde;
-- should fail
CREATE TABLE t1 (n INT) USING tde_heap;
-- should work
CREATE TABLE t2 (n INT) USING heap;
DROP TABLE t2;
DROP EXTENSION pg_tde;

@ -0,0 +1,54 @@
CREATE EXTENSION IF NOT EXISTS pg_tde;
SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault');
SET default_table_access_method = "tde_heap";
CREATE TABLE t1(n integer);
SELECT pg_tde_is_encrypted('t1');
VACUUM FULL t1;
SELECT pg_tde_is_encrypted('t1');
CREATE TABLE test_tab1 AS SELECT generate_series(1,10) a;
CREATE INDEX test_idx1 ON test_tab1(a);
SELECT pg_tde_is_encrypted('test_tab1');
SELECT pg_tde_is_encrypted('test_idx1');
REINDEX index CONCURRENTLY test_idx1;
SELECT pg_tde_is_encrypted('test_tab1');
SELECT pg_tde_is_encrypted('test_idx1');
CREATE TABLE mvtest_t (id int NOT NULL PRIMARY KEY, type text NOT NULL, amt numeric NOT NULL);
INSERT INTO mvtest_t VALUES
(1, 'x', 2),
(2, 'x', 3),
(3, 'y', 5),
(4, 'y', 7),
(5, 'z', 11);
CREATE MATERIALIZED VIEW mvtest_tm AS SELECT type, sum(amt) AS totamt FROM mvtest_t GROUP BY type WITH NO DATA;
SELECT pg_tde_is_encrypted('mvtest_tm');
REFRESH MATERIALIZED VIEW mvtest_tm;
SELECT pg_tde_is_encrypted('mvtest_tm');
CREATE TYPE rewritetype AS (a int);
CREATE TABLE rewritemetoo1 OF rewritetype;
CREATE TABLE rewritemetoo2 OF rewritetype;
SELECT pg_tde_is_encrypted('rewritemetoo1');
SELECT pg_tde_is_encrypted('rewritemetoo2');
ALTER TYPE rewritetype ALTER ATTRIBUTE a TYPE text cascade;
SELECT pg_tde_is_encrypted('rewritemetoo1');
SELECT pg_tde_is_encrypted('rewritemetoo2');
CREATE TABLE encrypted_table (
id SERIAL,
data TEXT,
created_at DATE NOT NULL,
PRIMARY KEY (id, created_at)
) USING tde_heap;
CREATE INDEX idx_date ON encrypted_table (created_at);
SELECT pg_tde_is_encrypted('encrypted_table');
CLUSTER encrypted_table USING idx_date;
SELECT pg_tde_is_encrypted('encrypted_table');
DROP EXTENSION pg_tde CASCADE;
RESET default_table_access_method;

@ -0,0 +1,3 @@
CREATE EXTENSION pg_tde;
SELECT pg_tde_version();
DROP EXTENSION pg_tde;

@ -113,6 +113,9 @@ RelKeyCache tde_rel_key_cache = {
};
/*
* TODO: WAL should have its own RelKeyCache
*/
static WALKeyCacheRec *tde_wal_key_cache = NULL;
static WALKeyCacheRec *tde_wal_key_last_rec = NULL;
@ -140,6 +143,7 @@ static off_t pg_tde_write_one_map_entry(int fd, const RelFileLocator *rlocator,
static bool pg_tde_delete_map_entry(const RelFileLocator *rlocator, uint32 key_type, char *db_map_path, off_t offset);
static int keyrotation_init_file(TDEPrincipalKeyInfo *new_principal_key_info, char *rotated_filename, char *filename, off_t *curr_pos);
static void finalize_key_rotation(const char *path_old, const char *path_new);
static void update_wal_keys_cache(void);
InternalKey *
pg_tde_create_smgr_key(const RelFileLocatorBackend *newrlocator)
@ -253,7 +257,7 @@ pg_tde_create_wal_key(InternalKey *rel_key_data, const RelFileLocator *newrlocat
{
ereport(ERROR,
(errmsg("principal key not configured"),
errhint("create one using pg_tde_set_principal_key before using encrypted WAL")));
errhint("create one using pg_tde_set_server_principal_key before using encrypted WAL")));
}
/* TODO: no need in generating key if TDE_KEY_TYPE_WAL_UNENCRYPTED */
@ -763,6 +767,8 @@ pg_tde_wal_last_key_set_lsn(XLogRecPtr lsn, const char *keyfile_path)
off_t write_pos,
last_key_idx;
LWLockAcquire(lock_pk, LW_EXCLUSIVE);
fd = BasicOpenFile(keyfile_path, O_RDWR | PG_BINARY);
if (fd < 0)
{
@ -775,7 +781,6 @@ pg_tde_wal_last_key_set_lsn(XLogRecPtr lsn, const char *keyfile_path)
last_key_idx = ((lseek(fd, 0, SEEK_END) - TDE_FILE_HEADER_SIZE) / MAP_ENTRY_DAT_SIZE) - 1;
write_pos = TDE_FILE_HEADER_SIZE + (last_key_idx * MAP_ENTRY_DAT_SIZE) + offsetof(TDEMapEntry, enc_key) + offsetof(InternalKey, start_lsn);
LWLockAcquire(lock_pk, LW_EXCLUSIVE);
/* TODO: pgstat_report_wait_start / pgstat_report_wait_end */
if (pg_pwrite(fd, &lsn, sizeof(XLogRecPtr), write_pos) != sizeof(XLogRecPtr))
{
@ -1230,6 +1235,25 @@ pg_tde_get_wal_cache_keys(void)
return tde_wal_key_cache;
}
/* Updates WAL keys cache pointers */
static void
update_wal_keys_cache(void)
{
WALKeyCacheRec *wal_rec = tde_wal_key_cache;
RelFileLocator rlocator = GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID);
for (int i = 0; i < tde_rel_key_cache.len && wal_rec; i++)
{
RelKeyCacheRec *rec = tde_rel_key_cache.data + i;
if (RelFileLocatorEquals(rec->locator, rlocator))
{
wal_rec->key = &rec->key;
wal_rec = wal_rec->next;
}
}
}
InternalKey *
pg_tde_read_last_wal_key(void)
{
@ -1466,6 +1490,9 @@ pg_tde_put_key_into_cache(const RelFileLocator *rlocator, InternalKey *key)
elog(WARNING, "could not mlock internal key cache pages: %m");
tde_rel_key_cache.cap = (size - 1) / sizeof(RelKeyCacheRec);
/* update wal key pointers after moving the cache */
update_wal_keys_cache();
}
rec = tde_rel_key_cache.data + tde_rel_key_cache.len;

@ -49,6 +49,14 @@ static const XLogSmgr tde_xlog_smgr = {
#ifndef FRONTEND
static Size TDEXLogEncryptBuffSize(void);
/*
* Must be the same as in replication/walsender.c
*
* This is used to calculate the encryption buffer size.
*/
#define MAX_SEND_SIZE (XLOG_BLCKSZ * 16)
static ssize_t TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count,
off_t offset, TimeLineID tli,
XLogSegNo segno);
@ -94,7 +102,7 @@ TDEXLogEncryptBuffSize(void)
int xbuffers;
xbuffers = (XLOGbuffers == -1) ? XLOGChooseNumBuffers() : XLOGbuffers;
return (Size) XLOG_BLCKSZ * xbuffers;
return Max(MAX_SEND_SIZE, mul_size(XLOG_BLCKSZ, xbuffers));
}
Size
@ -102,10 +110,11 @@ TDEXLogEncryptStateSize(void)
{
Size sz;
sz = TYPEALIGN(PG_IO_ALIGN_SIZE, TDEXLogEncryptBuffSize());
sz = add_size(sz, sizeof(EncryptionStateData));
sz = sizeof(EncryptionStateData);
sz = add_size(sz, TDEXLogEncryptBuffSize());
sz = add_size(sz, PG_IO_ALIGN_SIZE);
return MAXALIGN(sz);
return sz;
}
/*
@ -133,9 +142,14 @@ TDEXLogShmemInit(void)
TDEXLogEncryptStateSize(),
&foundBuf);
allocptr = ((char *) EncryptionState) + TYPEALIGN(PG_IO_ALIGN_SIZE, sizeof(EncryptionStateData));
memset(EncryptionState, 0, sizeof(EncryptionStateData));
allocptr = ((char *) EncryptionState) + sizeof(EncryptionStateData);
allocptr = (char *) TYPEALIGN(PG_IO_ALIGN_SIZE, allocptr);
EncryptionState->segBuf = allocptr;
Assert((char *) EncryptionState + TDEXLogEncryptStateSize() >= (char *) EncryptionState->segBuf + TDEXLogEncryptBuffSize());
pg_atomic_init_u64(&EncryptionState->enc_key_lsn, 0);
elog(DEBUG1, "pg_tde: initialized encryption buffer %lu bytes", TDEXLogEncryptStateSize());
@ -152,6 +166,8 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset,
InternalKey *key = &EncryptionKey;
char *enc_buff = EncryptionState->segBuf;
Assert(count <= TDEXLogEncryptBuffSize());
#ifdef TDE_XLOG_DEBUG
elog(DEBUG1, "write encrypted WAL, size: %lu, offset: %ld [%lX], seg: %X/%X, key_start_lsn: %X/%X",
count, offset, offset, LSN_FORMAT_ARGS(segno), LSN_FORMAT_ARGS(key->start_lsn));

@ -449,7 +449,7 @@ write_key_provider_info(KeyringProvideRecord *provider, Oid database_id,
seek_pos = before_pos;
break;
}
if (strcmp(existing_provider.provider_name, provider->provider_name) == 0)
if (strlen(existing_provider.provider_name) > 0 && strcmp(existing_provider.provider_name, provider->provider_name) == 0)
{
if (error_if_exists)
{
@ -633,7 +633,6 @@ scan_key_provider_file(ProviderScanType scanType, void *scanKey, Oid dbOid)
fd = BasicOpenFile(kp_info_path, PG_BINARY);
if (fd < 0)
{
fprintf(stderr, "WTF\n");
LWLockRelease(tde_provider_info_lock());
ereport(DEBUG2,
(errcode_for_file_access(),

@ -841,11 +841,6 @@ GetPrincipalKey(Oid dbOid, LWLockMode lockMode)
pg_tde_save_principal_key(&newPrincipalKey->keyInfo);
/* XLog the new use of the default key */
XLogBeginInsert();
XLogRegisterData((char *) &newPrincipalKey->keyInfo, sizeof(TDEPrincipalKeyInfo));
XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_PRINCIPAL_KEY);
push_principal_key_to_cache(newPrincipalKey);
pfree(newPrincipalKey);
@ -962,7 +957,7 @@ pg_tde_rotate_default_key_for_database(TDEPrincipalKey *oldKey, TDEPrincipalKey
newKey->keyInfo.databaseId = oldKey->keyInfo.databaseId;
/* key rotation */
is_rotated = pg_tde_perform_rotate_key(newKey, oldKey);
is_rotated = pg_tde_perform_rotate_key(oldKey, newKey);
if (is_rotated && (!TDEisInGlobalSpace(newKey->keyInfo.databaseId)))
{

@ -9,19 +9,25 @@
#include "postgres.h"
#include "nodes/parsenodes.h"
#include "access/transam.h"
typedef struct TdeCreateEvent
{
FullTransactionId tid; /* transaction id of the last event trigger,
* or 0 */
bool encryptMode; /* true when the table uses encryption */
Oid baseTableOid; /* Oid of table on which index is being
* created on. For create table statement this
* contains InvalidOid */
RangeVar *relation; /* Reference to the parsed relation from
* create statement */
bool alterSequenceMode; /* true when alter sequence is executed by
* pg_tde */
bool alterAccessMethodMode; /* during ALTER ... SET ACCESS METHOD,
* new file permissions shouldn't be
* based on earlier encryption status. */
} TdeCreateEvent;
extern TdeCreateEvent *GetCurrentTdeCreateEvent(void);
extern void validateCurrentEventTriggerState(bool mightStartTransaction);
#endif

@ -259,6 +259,8 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCode
return NULL;
}
memset(key->name, 0, sizeof(key->name));
memcpy(key->name, key_name, strnlen(key_name, sizeof(key->name) - 1));
memcpy(key->data.data, keyp, key->data.len);
free(keyp);
}

@ -275,6 +275,8 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCode
#endif
key = palloc_object(KeyInfo);
memset(key->name, 0, sizeof(key->name));
memcpy(key->name, key_name, strnlen(key_name, sizeof(key->name) - 1));
key->data.len = pg_b64_decode(responseKey, strlen(responseKey), (char *) key->data.data, MAX_KEY_DATA_SIZE);
if (key->data.len > MAX_KEY_DATA_SIZE)

@ -100,7 +100,14 @@ _PG_init(void)
{
if (!process_shared_preload_libraries_in_progress)
{
elog(ERROR, "pg_tde can only be loaded at server startup. Restart required.");
/*
* psql/pg_restore continue on error by default, and change access
* methods using set default_table_access_method. This error needs to
* be FATAL and close the connection, otherwise these tools will
* continue execution and create unencrypted tables when the intention
* was to make them encrypted.
*/
elog(FATAL, "pg_tde can only be loaded at server startup. Restart required.");
}
check_percona_api_version();
@ -148,6 +155,7 @@ pg_tde_extension_initialize(PG_FUNCTION_ARGS)
void
extension_install_redo(XLogExtensionInstall *xlrec)
{
pg_tde_init_data_dir();
run_extension_install_callbacks(xlrec, true);
}

@ -11,20 +11,29 @@
#include <stdarg.h>
#include <stdio.h>
/* version string we expect back from pg_tde_modify_key_provider */
#define PROGNAME "pg_tde_modify_key_provider (PostgreSQL) " PG_VERSION "\n"
/* version string we expect back from pg_tde_change_key_provider */
#define PROGNAME "pg_tde_change_key_provider (PostgreSQL) " PG_VERSION "\n"
static void
help(void)
{
printf("pg_tde_modify_key_provider changes the configuration of a pg_tde key provider");
puts("pg_tde_change_key_provider changes the configuration of a pg_tde key provider");
puts("");
printf("Usage:");
printf("pg_tde_modify_key_provider [-D <datadir>] <dbOid> <provider_name> file <filename>");
printf("pg_tde_modify_key_provider [-D <datadir>] <dbOid> <provider_name> vault <token> <url> <mount_path> [<ca_path>]");
printf("pg_tde_modify_key_provider [-D <datadir>] <dbOid> <provider_name> kmip <host> <port> <cert_path> [<ca_path>]");
printf("\nWARNING:");
printf("This tool only changes the values, without properly XLogging the changes. Only use it in case the database is inaccessible and can't be started.");
puts("Usage:");
puts("");
puts("pg_tde_change_key_provider [-D <datadir>] <dbOid> <provider_name> <new_provider_type> <provider_parameters...>");
puts("");
puts(" Where <new_provider_type> can be file, vault or kmip");
puts("");
puts("Depending on the provider type, the complete parameter list is:");
puts("");
puts("pg_tde_change_key_provider [-D <datadir>] <dbOid> <provider_name> file <filename>");
puts("pg_tde_change_key_provider [-D <datadir>] <dbOid> <provider_name> vault <token> <url> <mount_path> [<ca_path>]");
puts("pg_tde_change_key_provider [-D <datadir>] <dbOid> <provider_name> kmip <host> <port> <cert_path> [<ca_path>]");
puts("");
puts("WARNING:");
puts("");
puts("This tool only changes the values, without properly XLogging the changes, or adjusting the configuration in the running postgres processes. Only use it in case the database is inaccessible and can't be started.\n");
}
#define BUFFER_SIZE 1024
@ -74,7 +83,7 @@ build_json(char *buffer, int count,...)
}
if (ptr - buffer > BUFFER_SIZE)
{
printf("Error: Configuration too long.");
printf("Error: Configuration too long.\n");
return false;
}
}
@ -84,7 +93,7 @@ build_json(char *buffer, int count,...)
if (ptr - buffer > BUFFER_SIZE)
{
printf("Error: Configuration too long.");
printf("Error: Configuration too long.\n");
return false;
}
@ -113,7 +122,7 @@ main(int argc, char *argv[])
pg_logging_init(argv[0]);
pg_logging_set_level(PG_LOG_WARNING);
set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_tde_alter_key_provider"));
set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_tde_change_key_provider"));
if (argc > 1)
{
@ -124,7 +133,7 @@ main(int argc, char *argv[])
}
if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
{
puts("pg_tde_alter_key_provider (PostgreSQL) " PG_VERSION);
puts("pg_tde_change_key_provider (PostgreSQL) " PG_VERSION);
exit(0);
}
}
@ -143,11 +152,11 @@ main(int argc, char *argv[])
{
help();
puts("\n");
printf("Error: Data directory missing");
printf("Error: Data directory missing.\n");
exit(1);
}
if (argc - argstart < 3)
if (argc - argstart <= 3)
{
help();
exit(1);
@ -165,7 +174,7 @@ main(int argc, char *argv[])
{
help();
puts("\n");
printf("Error: wrong number of arguments");
printf("Error: wrong number of arguments.\n");
exit(1);
}
@ -183,7 +192,7 @@ main(int argc, char *argv[])
{
help();
puts("\n");
printf("Error: wrong number of arguments");
printf("Error: wrong number of arguments.\n");
exit(1);
}
@ -201,7 +210,7 @@ main(int argc, char *argv[])
{
help();
puts("\n");
printf("Error: wrong number of arguments");
printf("Error: wrong number of arguments.\n");
exit(1);
}
@ -215,7 +224,7 @@ main(int argc, char *argv[])
{
help();
puts("\n");
printf("Error: Unknown provider type: %s", new_provider_type);
printf("Error: Unknown provider type: %s\n.", new_provider_type);
exit(1);
}
@ -242,7 +251,7 @@ main(int argc, char *argv[])
if (keyring == NULL)
{
printf("Error: provider not found");
printf("Error: provider not found\n.");
exit(1);
}
@ -251,7 +260,7 @@ main(int argc, char *argv[])
provider.provider_type = get_keyring_provider_from_typename(new_provider_type);
modify_key_provider_info(&provider, db_oid, false);
printf("Key provider updated successfully!");
printf("Key provider updated successfully!\n");
return 0;
}

@ -31,9 +31,8 @@
#include "executor/spi.h"
/* Global variable that gets set at ddl start and cleard out at ddl end*/
static TdeCreateEvent tdeCurrentCreateEvent = {.relation = NULL};
static TdeCreateEvent tdeCurrentCreateEvent = {.tid = {.value = 0},.relation = NULL};
static bool alterSetAccessMethod = false;
static int event_trigger_level = 0;
static void reset_current_tde_create_event(void);
static Oid get_tde_table_am_oid(void);
@ -86,6 +85,23 @@ checkEncryptionClause(const char *accessMethod)
}
}
void
validateCurrentEventTriggerState(bool mightStartTransaction)
{
FullTransactionId tid = mightStartTransaction ? GetCurrentFullTransactionId() : GetCurrentFullTransactionIdIfAny();
if (RecoveryInProgress())
{
reset_current_tde_create_event();
return;
}
if (tdeCurrentCreateEvent.tid.value != InvalidFullTransactionId.value && tid.value != tdeCurrentCreateEvent.tid.value)
{
/* There was a failed query, end event trigger didn't execute */
reset_current_tde_create_event();
}
}
/*
* pg_tde_ddl_command_start_capture is an event trigger function triggered
* at the start of any DDL command execution.
@ -105,8 +121,6 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS)
EventTriggerData *trigdata;
Node *parsetree;
event_trigger_level++;
/* Ensure this function is being called as an event trigger */
if (!CALLED_AS_EVENT_TRIGGER(fcinfo)) /* internal error */
ereport(ERROR,
@ -115,16 +129,14 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS)
trigdata = (EventTriggerData *) fcinfo->context;
parsetree = trigdata->parsetree;
if (event_trigger_level == 1)
{
reset_current_tde_create_event();
}
if (IsA(parsetree, IndexStmt))
{
IndexStmt *stmt = (IndexStmt *) parsetree;
Oid relationId = RangeVarGetRelid(stmt->relation, NoLock, true);
validateCurrentEventTriggerState(true);
tdeCurrentCreateEvent.tid = GetCurrentFullTransactionId();
tdeCurrentCreateEvent.baseTableOid = relationId;
tdeCurrentCreateEvent.relation = stmt->relation;
@ -155,6 +167,10 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS)
CreateStmt *stmt = (CreateStmt *) parsetree;
const char *accessMethod = stmt->accessMethod;
validateCurrentEventTriggerState(true);
tdeCurrentCreateEvent.tid = GetCurrentFullTransactionId();
tdeCurrentCreateEvent.relation = stmt->relation;
checkEncryptionClause(accessMethod);
@ -164,6 +180,9 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS)
CreateTableAsStmt *stmt = (CreateTableAsStmt *) parsetree;
const char *accessMethod = stmt->into->accessMethod;
validateCurrentEventTriggerState(true);
tdeCurrentCreateEvent.tid = GetCurrentFullTransactionId();
tdeCurrentCreateEvent.relation = stmt->into->rel;
checkEncryptionClause(accessMethod);
@ -174,6 +193,9 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS)
ListCell *lcmd;
Oid relationId = RangeVarGetRelid(stmt->relation, NoLock, true);
validateCurrentEventTriggerState(true);
tdeCurrentCreateEvent.tid = GetCurrentFullTransactionId();
foreach(lcmd, stmt->cmds)
{
AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lcmd);
@ -184,6 +206,7 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS)
tdeCurrentCreateEvent.relation = stmt->relation;
tdeCurrentCreateEvent.baseTableOid = relationId;
tdeCurrentCreateEvent.alterAccessMethodMode = true;
checkEncryptionClause(accessMethod);
alterSetAccessMethod = true;
@ -222,7 +245,19 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS)
}
}
}
}
else
{
if (!tdeCurrentCreateEvent.alterAccessMethodMode)
{
/*
* Any other type of statement doesn't need TDE mode, except
* during alter access method. To make sure that we have no
* leftover setting from a previous error or something, we just
* reset the status here.
*/
reset_current_tde_create_event();
}
}
PG_RETURN_NULL();
}
@ -234,12 +269,18 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS)
Datum
pg_tde_ddl_command_end_capture(PG_FUNCTION_ARGS)
{
EventTriggerData *trigdata;
Node *parsetree;
trigdata = (EventTriggerData *) fcinfo->context;
parsetree = trigdata->parsetree;
/* Ensure this function is being called as an event trigger */
if (!CALLED_AS_EVENT_TRIGGER(fcinfo)) /* internal error */
ereport(ERROR,
(errmsg("Function can only be fired by event trigger manager")));
if (alterSetAccessMethod && !tdeCurrentCreateEvent.alterSequenceMode)
if (IsA(parsetree, AlterTableStmt) && tdeCurrentCreateEvent.alterAccessMethodMode)
{
/*
* sequences are not updated automatically call a helper function that
@ -261,9 +302,9 @@ pg_tde_ddl_command_end_capture(PG_FUNCTION_ARGS)
args[0] = ObjectIdGetDatum(tdeCurrentCreateEvent.baseTableOid);
nulls[0] = ' ';
tdeCurrentCreateEvent.alterSequenceMode = true;
ret = SPI_execute_plan(plan, args, nulls, false, 0);
tdeCurrentCreateEvent.alterSequenceMode = false;
tdeCurrentCreateEvent.alterAccessMethodMode = false;
SPI_finish();
@ -273,10 +314,16 @@ pg_tde_ddl_command_end_capture(PG_FUNCTION_ARGS)
}
}
event_trigger_level--;
/* All we need to do is to clear the event state */
reset_current_tde_create_event();
/*
* All we need to do is to clear the event state. Except when we are in
* alter access method mode, because during that, we have multiple nested
* event trigger running. Reset should only be called in the end, when it
* is set to false.
*/
if (!tdeCurrentCreateEvent.alterAccessMethodMode)
{
reset_current_tde_create_event();
}
PG_RETURN_NULL();
}
@ -287,7 +334,9 @@ reset_current_tde_create_event(void)
tdeCurrentCreateEvent.encryptMode = false;
tdeCurrentCreateEvent.baseTableOid = InvalidOid;
tdeCurrentCreateEvent.relation = NULL;
tdeCurrentCreateEvent.tid = InvalidFullTransactionId;
alterSetAccessMethod = false;
tdeCurrentCreateEvent.alterAccessMethodMode = false;
}
static Oid

@ -205,6 +205,14 @@ tde_mdcreate(RelFileLocator relold, SMgrRelation reln, ForkNumber forknum, bool
InternalKey *key;
TdeCreateEvent *event = GetCurrentTdeCreateEvent();
/*
* Make sure that even if a statement failed, and an event trigger end
* trigger didn't fire, we don't accidentaly create encrypted files when
* we don't have to. event above is a pointer, so it will reflect the
* correct state even if this changes it.
*/
validateCurrentEventTriggerState(false);
/*
* This is the only function that gets called during actual CREATE
* TABLE/INDEX (EVENT TRIGGER)
@ -213,20 +221,30 @@ tde_mdcreate(RelFileLocator relold, SMgrRelation reln, ForkNumber forknum, bool
mdcreate(relold, reln, forknum, isRedo);
/*
* Later calls then decide to encrypt or not based on the existence of the
* key
*/
key = tde_smgr_get_key(reln, event->alterSequenceMode ? NULL : &relold, true);
if (key)
if (forknum == MAIN_FORKNUM || forknum == INIT_FORKNUM)
{
tdereln->encrypted_relation = true;
tdereln->relKey = *key;
}
else
{
tdereln->encrypted_relation = false;
/*
* Only create keys when creating the main/init fork. Other forks can
* be created later, even during tde creation events. We definitely do
* not want to create keys then, even later, when we encrypt all
* forks!
*/
/*
* Later calls then decide to encrypt or not based on the existence of
* the key
*/
key = tde_smgr_get_key(reln, event->alterAccessMethodMode ? NULL : &relold, true);
if (key)
{
tdereln->encrypted_relation = true;
tdereln->relKey = *key;
}
else
{
tdereln->encrypted_relation = false;
}
}
}

@ -210,7 +210,8 @@ Boot_CreateStmt:
true,
&relfrozenxid,
&relminmxid,
true);
true,
NULL);
elog(DEBUG4, "bootstrap relation created");
}
else

@ -300,7 +300,8 @@ heap_create(const char *relname,
bool allow_system_table_mods,
TransactionId *relfrozenxid,
MultiXactId *relminmxid,
bool create_storage)
bool create_storage,
RelFileLocator *old_rlocator)
{
Relation rel;
@ -385,14 +386,31 @@ heap_create(const char *relname,
*/
if (create_storage)
{
RelFileLocator prev_rlocator = rel->rd_locator;
RelFileLocator new_rlocator = rel->rd_locator;
if (old_rlocator != NULL)
{
prev_rlocator = *old_rlocator;
/*
* table_relation_set_new_filelocator() takes old_rlocator from
* rel->rd_locator
*/
rel->rd_locator = prev_rlocator;
}
if (RELKIND_HAS_TABLE_AM(rel->rd_rel->relkind))
table_relation_set_new_filelocator(rel, &rel->rd_locator,
table_relation_set_new_filelocator(rel, &new_rlocator,
relpersistence,
relfrozenxid, relminmxid);
else if (RELKIND_HAS_STORAGE(rel->rd_rel->relkind))
RelationCreateStorage(rel->rd_locator, rel->rd_locator, relpersistence, true);
RelationCreateStorage(prev_rlocator, new_rlocator, relpersistence, true);
else
Assert(false);
/* restore the orginal rel's locator */
rel->rd_locator = new_rlocator;
}
/*
@ -1129,6 +1147,8 @@ heap_create_with_catalog(const char *relname,
Oid existing_relid;
Oid old_type_oid;
Oid new_type_oid;
RelFileLocator *old_rlocator = NULL;
Relation old_rel;
/* By default set to InvalidOid unless overridden by binary-upgrade */
RelFileNumber relfilenumber = InvalidRelFileNumber;
@ -1283,6 +1303,12 @@ heap_create_with_catalog(const char *relname,
else
relacl = NULL;
if (relrewrite != InvalidOid)
{
old_rel = table_open(relrewrite, AccessShareLock);
old_rlocator = &old_rel->rd_locator;
}
/*
* Create the relcache entry (mostly dummy at this point) and the physical
* disk file. (If we fail further down, it's the smgr's responsibility to
@ -1306,7 +1332,11 @@ heap_create_with_catalog(const char *relname,
allow_system_table_mods,
&relfrozenxid,
&relminmxid,
true);
true,
old_rlocator);
if (relrewrite != InvalidOid)
table_close(old_rel, AccessShareLock);
Assert(relid == RelationGetRelid(new_rel_desc));

@ -741,7 +741,8 @@ index_create(Relation heapRelation,
bits16 constr_flags,
bool allow_system_table_mods,
bool is_internal,
Oid *constraintId)
Oid *constraintId,
RelFileLocator *old_rlocator)
{
Oid heapRelationId = RelationGetRelid(heapRelation);
Relation pg_class;
@ -985,7 +986,8 @@ index_create(Relation heapRelation,
allow_system_table_mods,
&relfrozenxid,
&relminmxid,
create_storage);
create_storage,
old_rlocator);
Assert(relfrozenxid == InvalidTransactionId);
Assert(relminmxid == InvalidMultiXactId);
@ -1459,7 +1461,8 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId,
0,
true, /* allow table to be a system catalog? */
false, /* is_internal? */
NULL);
NULL,
&indexRelation->rd_locator);
/* Close the relations used and clean up */
index_close(indexRelation, NoLock);

@ -325,7 +325,7 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid,
BTREE_AM_OID,
rel->rd_rel->reltablespace,
collationIds, opclassIds, NULL, coloptions, NULL, (Datum) 0,
INDEX_CREATE_IS_PRIMARY, 0, true, true, NULL);
INDEX_CREATE_IS_PRIMARY, 0, true, true, NULL, NULL);
table_close(toast_rel, NoLock);

@ -1208,7 +1208,7 @@ DefineIndex(Oid tableId,
coloptions, NULL, reloptions,
flags, constr_flags,
allowSystemTableMods, !check_rights,
&createdConstraintId);
&createdConstraintId, NULL);
ObjectAddressSet(address, RelationRelationId, indexRelationId);

@ -0,0 +1 @@
/pg_tde_change_key_provider

@ -17,6 +17,7 @@
#include "catalog/indexing.h"
#include "catalog/objectaddress.h"
#include "parser/parse_node.h"
#include "storage/relfilelocator.h"
/* flag bits for CheckAttributeType/CheckAttributeNamesTypes */
@ -60,7 +61,8 @@ extern Relation heap_create(const char *relname,
bool allow_system_table_mods,
TransactionId *relfrozenxid,
MultiXactId *relminmxid,
bool create_storage);
bool create_storage,
RelFileLocator *old_rlocator);
extern Oid heap_create_with_catalog(const char *relname,
Oid relnamespace,

@ -16,6 +16,7 @@
#include "catalog/objectaddress.h"
#include "nodes/execnodes.h"
#include "storage/relfilelocator.h"
#define DEFAULT_INDEX_TYPE "btree"
@ -86,7 +87,8 @@ extern Oid index_create(Relation heapRelation,
bits16 constr_flags,
bool allow_system_table_mods,
bool is_internal,
Oid *constraintId);
Oid *constraintId,
RelFileLocator *old_rlocator);
#define INDEX_CONSTR_CREATE_MARK_AS_PRIMARY (1 << 0)
#define INDEX_CONSTR_CREATE_DEFERRABLE (1 << 1)

Loading…
Cancel
Save