Vault access token must be always in file

pull/230/head
Andrew Pogrebnoy 4 months ago committed by Andrew Pogrebnoi
parent 591d6048e0
commit 356f619ad2
  1. 7
      ci_scripts/setup-keyring-servers.sh
  2. 2
      contrib/pg_tde/documentation/docs/command-line-tools/pg-tde-change-key-provider.md
  3. 12
      contrib/pg_tde/documentation/docs/functions.md
  4. 6
      contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md
  5. 4
      contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md
  6. 4
      contrib/pg_tde/documentation/docs/wal-encryption.md
  7. 7
      contrib/pg_tde/expected/vault_v2_test.out
  8. 32
      contrib/pg_tde/pg_tde--1.0-rc.sql
  9. 8
      contrib/pg_tde/sql/vault_v2_test.sql
  10. 37
      contrib/pg_tde/src/catalog/tde_keyring.c
  11. 12
      contrib/pg_tde/src/catalog/tde_keyring_parse_opts.c
  12. 1
      contrib/pg_tde/src/include/keyring/keyring_api.h
  13. 4
      contrib/pg_tde/src/pg_tde_change_key_provider.c
  14. 8
      contrib/pg_tde/t/016_pg_tde_change_key_provider.pl

@ -20,8 +20,5 @@ pykmip-server -f "$SCRIPT_DIR/../contrib/pg_tde/pykmip-server.conf" -l /tmp/kmip
TV=$(mktemp)
{ exec >$TV; vault server -dev; } &
sleep 10
export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
echo "export ROOT_TOKEN=$ROOT_TOKEN"
if [ -v GITHUB_ACTIONS ]; then
echo "ROOT_TOKEN=$ROOT_TOKEN" >> $GITHUB_ENV
fi
echo $(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) > /tmp/vault_test_token
echo "write vault token to /tmp/vault_test_token"

@ -31,6 +31,6 @@ Depending on the provider type, the additional parameters are:
```bash
pg_tde_change_key_provider [-D <datadir>] <dbOid> <provider_name> file <filename>
pg_tde_change_key_provider [-D <datadir>] <dbOid> <provider_name> vault <token> <url> <mount_path> [<ca_path>]
pg_tde_change_key_provider [-D <datadir>] <dbOid> <provider_name> vault <token_path> <url> <mount_path> [<ca_path>]
pg_tde_change_key_provider [-D <datadir>] <dbOid> <provider_name> kmip <host> <port> <cert_path> <key_path> [<ca_path>]
```

@ -63,14 +63,14 @@ Use the following functions to add the Vault provider:
```sql
SELECT pg_tde_add_database_key_provider_vault_v2(
'provider-name',
'secret_token',
'secret_token_path',
'url','mount',
'ca_path'
);
SELECT pg_tde_add_global_key_provider_vault_v2(
'provider-name',
'secret_token',
'secret_token_path',
'url','mount',
'ca_path'
);
@ -81,7 +81,7 @@ These functions change the Vault provider:
```sql
SELECT pg_tde_change_database_key_provider_vault_v2(
'provider-name',
'secret_token',
'secret_token_path',
'url',
'mount',
'ca_path'
@ -89,7 +89,7 @@ SELECT pg_tde_change_database_key_provider_vault_v2(
SELECT pg_tde_change_global_key_provider_vault_v2(
'provider-name',
'secret_token',
'secret_token_path',
'url',
'mount',
'ca_path'
@ -101,13 +101,11 @@ where:
* `provider-name` is the name of the key provider
* `url` is the URL of the Vault server
* `mount` is the mount point on the Vault server where the key provider should store the keys
* `secret_token` is an access token with read and write access to the above mount point
* `secret_token_path` is a path to the file that contains an access token with read and write access to the above mount point
* **[optional]** `ca_path` is the path of the CA file used for SSL verification
All parameters can be either strings, or JSON objects [referencing remote parameters](how-to/external-parameters.md).
!!! important
Never specify the secret token directly, use a remote parameter instead.
#### Adding or modifying KMIP providers

@ -10,7 +10,7 @@ You can configure `pg_tde` to use HashiCorp Vault as a global key provider for m
```sql
SELECT pg_tde_add_global_key_provider_vault_v2(
'provider-name',
'secret_token',
'secret_token_path',
'url',
'mount',
'ca_path'
@ -20,7 +20,7 @@ SELECT pg_tde_add_global_key_provider_vault_v2(
## Parameter descriptions
* `provider-name` is the name to identify this key provider
* `secret_token` is an access token with read and write access to the above mount point
* `secret_token_path` is a path to the file that contains an access token with read and write access to the above mount point
* `url` is the URL of the Vault server
* `mount` is the mount point where the keyring should store the keys
* [optional] `ca_path` is the path of the CA file used for SSL verification
@ -30,7 +30,7 @@ The following example is for testing purposes only. Use secure tokens and proper
```sql
SELECT pg_tde_add_global_key_provider_vault_v2(
'my-vault',
'hvs.zPuyktykA...example...ewUEnIRVaKoBzs2',
'/path/to/token_file',
'http://vault.vault.svc.cluster.local:8200',
'secret/data',
NULL

@ -84,14 +84,14 @@ You must do these steps for every database where you have created the extension.
The Vault server setup is out of scope of this document.
```sql
SELECT pg_tde_add_database_key_provider_vault_v2('provider-name','secret_token','url','mount','ca_path');
SELECT pg_tde_add_database_key_provider_vault_v2('provider-name','secret_token_path','url','mount','ca_path');
```
where:
* `url` is the URL of the Vault server
* `mount` is the mount point where the keyring should store the keys
* `secret_token` is an access token with read and write access to the above mount point
* `secret_token_path` is a path to the file that contains an access token with read and write access to the above mount point
* [optional] `ca_path` is the path of the CA file used for SSL verification
<i warning>:material-information: Warning:</i> This example is for testing purposes only:

@ -40,7 +40,7 @@ Before turning WAL encryption on, you must follow the steps below to create your
=== "With HashiCorp Vault"
```sql
SELECT pg_tde_add_global_key_provider_vault_v2('provider-name', 'secret_token', 'url', 'mount', 'ca_path');
SELECT pg_tde_add_global_key_provider_vault_v2('provider-name', 'secret_token_path', 'url', 'mount', 'ca_path');
```
where:
@ -48,7 +48,7 @@ Before turning WAL encryption on, you must follow the steps below to create your
* `provider-name` is the name you define for the key provider
* `url` is the URL of the Vault server
* `mount` is the mount point where the keyring should store the keys
* `secret_token` is an access token with read and write access to the above mount point
* `secret_token_path` is a path to the file that contains an access token with read and write access to the above mount point
* [optional] `ca_path` is the path of the CA file used for SSL verification
=== "With keyring file"

@ -1,6 +1,5 @@
CREATE EXTENSION IF NOT EXISTS pg_tde;
\getenv root_token ROOT_TOKEN
SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect',:'root_token','http://127.0.0.1:8200','DUMMY-TOKEN',NULL);
SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect','/tmp/vault_test_token','http://127.0.0.1:8200','DUMMY-TOKEN',NULL);
pg_tde_add_database_key_provider_vault_v2
-------------------------------------------
@ -16,7 +15,7 @@ CREATE TABLE test_enc(
) USING tde_heap;
ERROR: principal key not configured
HINT: create one using pg_tde_set_key before using encrypted tables
SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2',:'root_token','http://127.0.0.1:8200','secret',NULL);
SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2','/tmp/vault_test_token','http://127.0.0.1:8200','secret',NULL);
pg_tde_add_database_key_provider_vault_v2
-------------------------------------------
@ -52,6 +51,6 @@ SELECT pg_tde_verify_key();
DROP TABLE test_enc;
-- Creating provider fails if we can't connect to vault
SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token', 'http://127.0.0.1:61', 'secret', NULL);
SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', '/tmp/vault_test_token', 'http://127.0.0.1:61', 'secret', NULL);
ERROR: HTTP(S) request to keyring provider "will-not-work" failed
DROP EXTENSION pg_tde;

@ -31,7 +31,7 @@ BEGIN ATOMIC
END;
CREATE FUNCTION pg_tde_add_database_key_provider_vault_v2(provider_name TEXT,
vault_token TEXT,
vault_token_path TEXT,
vault_url TEXT,
vault_mount_path TEXT,
vault_ca_path TEXT)
@ -42,13 +42,13 @@ BEGIN ATOMIC
-- load_vaultV2_keyring_provider_options function.
SELECT pg_tde_add_database_key_provider('vault-v2', provider_name,
json_object('url' VALUE COALESCE(vault_url, ''),
'token' VALUE COALESCE(vault_token, ''),
'tokenPath' VALUE COALESCE(vault_token_path, ''),
'mountPath' VALUE COALESCE(vault_mount_path, ''),
'caPath' VALUE COALESCE(vault_ca_path, '')));
END;
CREATE FUNCTION pg_tde_add_database_key_provider_vault_v2(provider_name TEXT,
vault_token JSON,
vault_token_path JSON,
vault_url JSON,
vault_mount_path JSON,
vault_ca_path JSON)
@ -59,7 +59,7 @@ BEGIN ATOMIC
-- load_vaultV2_keyring_provider_options function.
SELECT pg_tde_add_database_key_provider('vault-v2', provider_name,
json_object('url' VALUE vault_url,
'token' VALUE vault_token,
'tokenPath' VALUE vault_token_path,
'mountPath' VALUE vault_mount_path,
'caPath' VALUE vault_ca_path));
END;
@ -150,7 +150,7 @@ BEGIN ATOMIC
END;
CREATE FUNCTION pg_tde_add_global_key_provider_vault_v2(provider_name TEXT,
vault_token TEXT,
vault_token_path TEXT,
vault_url TEXT,
vault_mount_path TEXT,
vault_ca_path TEXT)
@ -161,13 +161,13 @@ BEGIN ATOMIC
-- load_vaultV2_keyring_provider_options function.
SELECT pg_tde_add_global_key_provider('vault-v2', provider_name,
json_object('url' VALUE COALESCE(vault_url, ''),
'token' VALUE COALESCE(vault_token, ''),
'tokenPath' VALUE COALESCE(vault_token_path, ''),
'mountPath' VALUE COALESCE(vault_mount_path, ''),
'caPath' VALUE COALESCE(vault_ca_path, '')));
END;
CREATE FUNCTION pg_tde_add_global_key_provider_vault_v2(provider_name TEXT,
vault_token JSON,
vault_token_path JSON,
vault_url JSON,
vault_mount_path JSON,
vault_ca_path JSON)
@ -178,7 +178,7 @@ BEGIN ATOMIC
-- load_vaultV2_keyring_provider_options function.
SELECT pg_tde_add_global_key_provider('vault-v2', provider_name,
json_object('url' VALUE vault_url,
'token' VALUE vault_token,
'tokenPath' VALUE vault_token_path,
'mountPath' VALUE vault_mount_path,
'caPath' VALUE vault_ca_path));
END;
@ -249,7 +249,7 @@ BEGIN ATOMIC
END;
CREATE FUNCTION pg_tde_change_database_key_provider_vault_v2(provider_name TEXT,
vault_token TEXT,
vault_token_path TEXT,
vault_url TEXT,
vault_mount_path TEXT,
vault_ca_path TEXT)
@ -260,13 +260,13 @@ BEGIN ATOMIC
-- load_vaultV2_keyring_provider_options function.
SELECT pg_tde_change_database_key_provider('vault-v2', provider_name,
json_object('url' VALUE COALESCE(vault_url, ''),
'token' VALUE COALESCE(vault_token, ''),
'tokenPath' VALUE COALESCE(vault_token_path, ''),
'mountPath' VALUE COALESCE(vault_mount_path, ''),
'caPath' VALUE COALESCE(vault_ca_path, '')));
END;
CREATE FUNCTION pg_tde_change_database_key_provider_vault_v2(provider_name TEXT,
vault_token JSON,
vault_token_path JSON,
vault_url JSON,
vault_mount_path JSON,
vault_ca_path JSON)
@ -277,7 +277,7 @@ BEGIN ATOMIC
-- load_vaultV2_keyring_provider_options function.
SELECT pg_tde_change_database_key_provider('vault-v2', provider_name,
json_object('url' VALUE vault_url,
'token' VALUE vault_token,
'tokenPath' VALUE vault_token_path,
'mountPath' VALUE vault_mount_path,
'caPath' VALUE vault_ca_path));
END;
@ -348,7 +348,7 @@ BEGIN ATOMIC
END;
CREATE FUNCTION pg_tde_change_global_key_provider_vault_v2(provider_name TEXT,
vault_token TEXT,
vault_token_path TEXT,
vault_url TEXT,
vault_mount_path TEXT,
vault_ca_path TEXT)
@ -359,13 +359,13 @@ BEGIN ATOMIC
-- load_vaultV2_keyring_provider_options function.
SELECT pg_tde_change_global_key_provider('vault-v2', provider_name,
json_object('url' VALUE COALESCE(vault_url, ''),
'token' VALUE COALESCE(vault_token, ''),
'tokenPath' VALUE COALESCE(vault_token_path, ''),
'mountPath' VALUE COALESCE(vault_mount_path, ''),
'caPath' VALUE COALESCE(vault_ca_path, '')));
END;
CREATE FUNCTION pg_tde_change_global_key_provider_vault_v2(provider_name TEXT,
vault_token JSON,
vault_token_path JSON,
vault_url JSON,
vault_mount_path JSON,
vault_ca_path JSON)
@ -376,7 +376,7 @@ BEGIN ATOMIC
-- load_vaultV2_keyring_provider_options function.
SELECT pg_tde_change_global_key_provider('vault-v2', provider_name,
json_object('url' VALUE vault_url,
'token' VALUE vault_token,
'tokenPath' VALUE vault_token_path,
'mountPath' VALUE vault_mount_path,
'caPath' VALUE vault_ca_path));
END;

@ -1,8 +1,6 @@
CREATE EXTENSION IF NOT EXISTS pg_tde;
\getenv root_token ROOT_TOKEN
SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect',:'root_token','http://127.0.0.1:8200','DUMMY-TOKEN',NULL);
SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect','/tmp/vault_test_token','http://127.0.0.1:8200','DUMMY-TOKEN',NULL);
-- FAILS
SELECT pg_tde_set_key_using_database_key_provider('vault-v2-key','vault-incorrect');
@ -12,7 +10,7 @@ CREATE TABLE test_enc(
PRIMARY KEY (id)
) USING tde_heap;
SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2',:'root_token','http://127.0.0.1:8200','secret',NULL);
SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2','/tmp/vault_test_token','http://127.0.0.1:8200','secret',NULL);
SELECT pg_tde_set_key_using_database_key_provider('vault-v2-key','vault-v2');
CREATE TABLE test_enc(
@ -32,6 +30,6 @@ SELECT pg_tde_verify_key();
DROP TABLE test_enc;
-- Creating provider fails if we can't connect to vault
SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token', 'http://127.0.0.1:61', 'secret', NULL);
SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', '/tmp/vault_test_token', 'http://127.0.0.1:61', 'secret', NULL);
DROP EXTENSION pg_tde;

@ -62,6 +62,7 @@ static GenericKeyring *load_keyring_provider_options(ProviderType provider_type,
static KmipKeyring *load_kmip_keyring_provider_options(char *keyring_options);
static VaultV2Keyring *load_vaultV2_keyring_provider_options(char *keyring_options);
static int open_keyring_infofile(Oid dbOid, int flags);
static char *get_file_value(const char *path, const char *field_name);
#ifdef FRONTEND
@ -870,19 +871,22 @@ load_vaultV2_keyring_provider_options(char *keyring_options)
(GenericKeyring *) vaultV2_keyring,
keyring_options, strlen(keyring_options));
if (vaultV2_keyring->vault_token == NULL || vaultV2_keyring->vault_token[0] == '\0' ||
if (vaultV2_keyring->vault_token_path == NULL || vaultV2_keyring->vault_token_path[0] == '\0' ||
vaultV2_keyring->vault_url == NULL || vaultV2_keyring->vault_url[0] == '\0' ||
vaultV2_keyring->vault_mount_path == NULL || vaultV2_keyring->vault_mount_path[0] == '\0')
{
ereport(WARNING,
errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("missing in the keyring options:%s%s%s",
(vaultV2_keyring->vault_token != NULL && vaultV2_keyring->vault_token[0] != '\0') ? "" : " token",
(vaultV2_keyring->vault_token_path != NULL && vaultV2_keyring->vault_token_path[0] != '\0') ? "" : " tokenPath",
(vaultV2_keyring->vault_url != NULL && vaultV2_keyring->vault_url[0] != '\0') ? "" : " url",
(vaultV2_keyring->vault_mount_path != NULL && vaultV2_keyring->vault_mount_path[0] != '\0') ? "" : " mountPath"));
return NULL;
}
/* TODO: the vault_token mem should be protected from paging to the swap */
vaultV2_keyring->vault_token = get_file_value(vaultV2_keyring->vault_token_path, "vault_token");
return vaultV2_keyring;
}
@ -916,6 +920,33 @@ load_kmip_keyring_provider_options(char *keyring_options)
return kmip_keyring;
}
#define MAX_FILE_DATA_LENGTH 1024
static char *
get_file_value(const char *path, const char *field_name)
{
int fd = -1;
char *val;
fd = BasicOpenFile(path, O_RDONLY);
if (fd < 0)
{
elog(ERROR, "failed to open file \"%s\" for \"%s\"", path, field_name);
}
val = palloc0(MAX_FILE_DATA_LENGTH);
if (pg_pread(fd, val, MAX_FILE_DATA_LENGTH, 0) == -1)
{
close(fd);
elog(ERROR, "failed to read file \"%s\" for \"%s\"", path, field_name);
}
/* remove trailing whitespace */
val[strcspn(val, " \t\n\r")] = '\0';
close(fd);
return val;
}
static void
debug_print_kerying(GenericKeyring *keyring)
{
@ -928,7 +959,7 @@ debug_print_kerying(GenericKeyring *keyring)
elog(DEBUG2, "File Keyring Path: %s", ((FileKeyring *) keyring)->file_name);
break;
case VAULT_V2_KEY_PROVIDER:
elog(DEBUG2, "Vault Keyring Token: %s", ((VaultV2Keyring *) keyring)->vault_token);
elog(DEBUG2, "Vault Keyring Token Path: %s", ((VaultV2Keyring *) keyring)->vault_token_path);
elog(DEBUG2, "Vault Keyring URL: %s", ((VaultV2Keyring *) keyring)->vault_url);
elog(DEBUG2, "Vault Keyring Mount Path: %s", ((VaultV2Keyring *) keyring)->vault_mount_path);
elog(DEBUG2, "Vault Keyring CA Path: %s", ((VaultV2Keyring *) keyring)->vault_ca_path);

@ -64,7 +64,7 @@ typedef enum JsonKeyringField
/* Settings specific for the individual key provider types. */
JK_FILE_PATH,
JK_VAULT_TOKEN,
JK_VAULT_TOKEN_PATH,
JK_VAULT_URL,
JK_VAULT_MOUNT_PATH,
JK_VAULT_CA_PATH,
@ -93,7 +93,7 @@ static const char *JK_FIELD_NAMES[JK_FIELDS_TOTAL] = {
*/
[JK_FILE_PATH] = "path",
[JK_VAULT_TOKEN] = "token",
[JK_VAULT_TOKEN_PATH] = "tokenPath",
[JK_VAULT_URL] = "url",
[JK_VAULT_MOUNT_PATH] = "mountPath",
[JK_VAULT_CA_PATH] = "caPath",
@ -358,8 +358,8 @@ json_kring_object_field_start(void *state, char *fname, bool isnull)
break;
case VAULT_V2_KEY_PROVIDER:
if (strcmp(fname, JK_FIELD_NAMES[JK_VAULT_TOKEN]) == 0)
parse->top_level_field = JK_VAULT_TOKEN;
if (strcmp(fname, JK_FIELD_NAMES[JK_VAULT_TOKEN_PATH]) == 0)
parse->top_level_field = JK_VAULT_TOKEN_PATH;
else if (strcmp(fname, JK_FIELD_NAMES[JK_VAULT_URL]) == 0)
parse->top_level_field = JK_VAULT_URL;
else if (strcmp(fname, JK_FIELD_NAMES[JK_VAULT_MOUNT_PATH]) == 0)
@ -498,8 +498,8 @@ json_kring_assign_scalar(JsonKeyringState *parse, JsonKeyringField field, char *
file->file_name = value;
break;
case JK_VAULT_TOKEN:
vault->vault_token = value;
case JK_VAULT_TOKEN_PATH:
vault->vault_token_path = value;
break;
case JK_VAULT_URL:
vault->vault_url = value;

@ -74,6 +74,7 @@ typedef struct VaultV2Keyring
{
GenericKeyring keyring; /* Must be the first field */
char *vault_token;
char *vault_token_path;
char *vault_url;
char *vault_ca_path;
char *vault_mount_path;

@ -27,7 +27,7 @@ help(void)
puts("Depending on the provider type, the complete parameter list is:");
puts("");
puts("pg_tde_change_key_provider [-D <datadir>] <dbOid> <provider_name> file <filename>");
puts("pg_tde_change_key_provider [-D <datadir>] <dbOid> <provider_name> vault-v2 <url> <token> <mount_path> [<ca_path>]");
puts("pg_tde_change_key_provider [-D <datadir>] <dbOid> <provider_name> vault-v2 <url> <token_path> <mount_path> [<ca_path>]");
puts("pg_tde_change_key_provider [-D <datadir>] <dbOid> <provider_name> kmip <host> <port> <cert_path> <key_path> [<ca_path>]");
puts("");
printf("Use dbOid %d for global key providers.\n", GLOBAL_DATA_TDE_OID);
@ -196,7 +196,7 @@ main(int argc, char *argv[])
if (!build_json(json, 4,
"url", argv[4 + argstart],
"token", argv[5 + argstart],
"tokenPath", argv[5 + argstart],
"mountPath", argv[6 + argstart],
"caPath", (argc - argstart > 7 ? argv[7 + argstart] : "")))
{

@ -64,7 +64,7 @@ command_like(
'database-provider',
'vault-v2',
'http://vault-server.example:8200/',
'secret-token',
'/tmp/vault_test_token',
'mount-path',
'/tmp/ca_path',
],
@ -85,8 +85,8 @@ $options = decode_json(
'postgres',
q{SELECT options FROM pg_tde_list_all_database_key_providers() WHERE provider_name = 'database-provider'}
));
is($options->{token}, 'secret-token',
'token is set correctly for vault-v2 provider');
is($options->{tokenPath}, '/tmp/vault_test_token',
'tokenPath is set correctly for vault-v2 provider');
is( $options->{url},
'http://vault-server.example:8200/',
'url is set correctly for vault-v2 provider');
@ -147,7 +147,7 @@ command_like(
'global-provider',
'vault-v2',
'http://vault-server.example:8200/',
'secret-token',
'/tmp/vault_test_token',
'mount-path',
'/tmp/ca_path',
],

Loading…
Cancel
Save