Updating CI scripts after recent changes (#129)

This commit removes outdated keyring configuration code, and
adds vault support to all runners.

Also contains fixes to the vault and file providers.

.github/workflows/postgresql-16-pgdg-package-pgxs.yml

@@ -27,6 +27,10 @@ jobs:
             libjson-c-dev libcurl4-openssl-dev
           sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y vault
+
 
       - name: Install PG Distribution Postgresql 16
         run: |
@@ -57,15 +61,20 @@ jobs:
 
       - name: Start pg_tde tests
         run: |
+          TV=$(mktemp)
+          { exec >$TV; vault server -dev; } &
+          sleep 10
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          echo "Root token: $ROOT_TOKEN"
+
+          sudo psql -V
+          
           sudo service postgresql stop
           echo "shared_preload_libraries = 'pg_tde'" |
             sudo tee -a /etc/postgresql/16/main/postgresql.conf
-          echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" |
-            sudo tee -a /etc/postgresql/16/main/postgresql.conf
-          cp keyring.json /tmp/keyring.json
           sudo service postgresql start
-          sudo psql -V
-          sudo -u postgres bash -c 'make installcheck USE_PGXS=1'
+
+          sudo -u postgres bash -c "export ROOT_TOKEN=$ROOT_TOKEN && make installcheck USE_PGXS=1"
         working-directory: src/pg_tde
 
       - name: Report on test fail

.github/workflows/postgresql-16-src-make-macos.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: macos-12
     steps:
       - name: Install dependencies
-        run: brew install json-c # All other required deps already installed in this image.
+        run: brew install json-c vault gnu-sed # All other required deps already installed in this image.
 
       - name: Clone postgres repository
         uses: actions/checkout@v4
@@ -47,13 +47,15 @@ jobs:
           initdb -D $HOME/pgsql/data
           echo "shared_preload_libraries = 'pg_tde'" >> \
             /$HOME/pgsql/data/postgresql.conf
-          echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" >> \
-            $HOME/pgsql/data/postgresql.conf
-          cp src/contrib/pg_tde/keyring.json /tmp/keyring.json
           pg_ctl -D $HOME/pgsql/data -l logfile start
 
       - name: Test pg_tde
         run: |
+          TV=$(mktemp)
+          { exec >$TV; vault server -dev; } &
+          sleep 10
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          echo "Root token: $ROOT_TOKEN"
           make installcheck
         working-directory: src/contrib/pg_tde
 

.github/workflows/postgresql-16-src-make-ssl11.yml

@@ -29,6 +29,9 @@ jobs:
             uuid-dev liblz4-dev libjson-c-dev libcurl4-openssl-dev
           sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y vault
 
       - name: Clone postgres repository
         uses: actions/checkout@v2
@@ -66,13 +69,15 @@ jobs:
           initdb -D /opt/pgsql/data
           echo "shared_preload_libraries = 'pg_tde'" >> \
             /opt/pgsql/data/postgresql.conf
-          echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" >> \
-            /opt/pgsql/data/postgresql.conf
-          cp src/contrib/pg_tde/keyring.json /tmp/keyring.json
           pg_ctl -D /opt/pgsql/data -l logfile start
 
       - name: Test pg_tde
         run: |
+          TV=$(mktemp)
+          { exec >$TV; vault server -dev; } &
+          sleep 10
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          echo "Root token: $ROOT_TOKEN"
           make installcheck
         working-directory: src/contrib/pg_tde
 

.github/workflows/postgresql-16-src-make.yml

@@ -29,6 +29,9 @@ jobs:
             uuid-dev liblz4-dev libjson-c-dev libcurl4-openssl-dev
           sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y vault
 
       - name: Clone postgres repository
         uses: actions/checkout@v2
@@ -66,13 +69,15 @@ jobs:
           initdb -D /opt/pgsql/data
           echo "shared_preload_libraries = 'pg_tde'" >> \
             /opt/pgsql/data/postgresql.conf
-          echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" >> \
-            /opt/pgsql/data/postgresql.conf
-          cp src/contrib/pg_tde/keyring.json /tmp/keyring.json
           pg_ctl -D /opt/pgsql/data -l logfile start
 
       - name: Test pg_tde
         run: |
+          TV=$(mktemp)
+          { exec >$TV; vault server -dev; } &
+          sleep 10
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          echo "Root token: $ROOT_TOKEN"
           make installcheck
         working-directory: src/contrib/pg_tde
 

.github/workflows/postgresql-16-src-meson-macos.yml

@@ -32,23 +32,13 @@ jobs:
           cd build && ninja && ninja install
         working-directory: src
 
-      - name: Test pg_tde with keyring_file
-        run: |
-          cp ../contrib/pg_tde/keyring.json /tmp/keyring.json
-          meson test --suite setup -v
-          meson test --suite pg_tde -v --num-processes 1
-        working-directory: src/build
-
-      - name: Test pg_tde with keyring_vault
+      - name: Test pg_tde
         run: |
           TV=$(mktemp)
           { exec >$TV; vault server -dev; } &
           sleep 10
-          ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
           echo "Root token: $ROOT_TOKEN"
-          cp ../contrib/pg_tde/keyring-vault.json /tmp/keyring.json
-          gsed -i "s/ROOT_TOKEN/$ROOT_TOKEN/g" /tmp/keyring.json
-          cat /tmp/keyring.json
           meson test --suite setup -v
           meson test --suite pg_tde -v --num-processes 1
         working-directory: src/build

.github/workflows/postgresql-16-src-meson-perf.yml

@@ -33,6 +33,10 @@ jobs:
             sysbench libcurl4-openssl-dev
           sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y vault
+
 
       - name: Clone postgres repository
         uses: actions/checkout@v2
@@ -58,7 +62,12 @@ jobs:
 
       - name: Test pg_tde
         run: |
-          cp ../contrib/pg_tde/keyring.json /tmp/keyring.json
+          TV=$(mktemp)
+          { exec >$TV; vault server -dev; } &
+          sleep 10
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          echo "Root token: $ROOT_TOKEN"
+          
           meson test --suite setup -v
           meson test --suite pg_tde -v --num-processes 1
         working-directory: src/build

.github/workflows/postgresql-16-src-meson.yml

@@ -56,32 +56,14 @@ jobs:
           cd build && ninja && ninja install
         working-directory: src
 
-      - name: Test pg_tde with keyring_file
-        run: |
-          cp ../contrib/pg_tde/keyring.json /tmp/keyring.json
-          meson test --suite setup -v
-          meson test --suite pg_tde -v --num-processes 1
-        working-directory: src/build
-
-      - name: Report on test fail
-        uses: actions/upload-artifact@v2
-        if: ${{ failure() }}
-        with:
-          name: Regressions diff and postgresql log
-          path: |
-            src/build/testrun/pg_tde/regress/
-          retention-days: 3
-
-      - name: Test pg_tde with keyring_vault
+      - name: Test pg_tde
         run: |
           TV=$(mktemp)
           { exec >$TV; vault server -dev; } &
           sleep 10
-          ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
           echo "Root token: $ROOT_TOKEN"
-          cp ../contrib/pg_tde/keyring-vault.json /tmp/keyring.json
-          sed -i "s/ROOT_TOKEN/$ROOT_TOKEN/g" /tmp/keyring.json
-          cat /tmp/keyring.json
+
           meson test --suite setup -v
           meson test --suite pg_tde -v --num-processes 1
         working-directory: src/build
@@ -94,4 +76,3 @@ jobs:
           path: |
             src/build/testrun/pg_tde/regress/
           retention-days: 3
-

Makefile.in

@@ -14,7 +14,8 @@ update_compare_indexes \
 pgtde_is_encrypted \
 multi_insert \
 trigger_on_view \
-insert_update_delete
+insert_update_delete \
+vault_v2_test
 TAP_TESTS = 1
 
 OBJS = src/encryption/enc_tde.o \

docker/Dockerfile

@@ -21,8 +21,6 @@ RUN cp /usr/share/postgresql/postgresql.conf.sample /etc/postgresql/postgresql.c
     echo "shared_preload_libraries = 'pg_tde'" >> /etc/postgresql/postgresql.conf; \
     #    echo "log_min_messages = debug3" >> /etc/postgresql/postgresql.conf; \
     #    echo "log_min_error_statement = debug3" >> /etc/postgresql/postgresql.conf; \
-    echo "pg_tde.keyringConfigFile = '/etc/postgresql/tde_conf.json'" >> /etc/postgresql/postgresql.conf; \
-    echo "{'provider': 'file','datafile': '/etc/postgresql/pgkeyring',}" > /etc/postgresql/tde_conf.json; \
     chown postgres /etc/postgresql/tde_conf.json; \
     mkdir -p /docker-entrypoint-initdb.d
 COPY ./docker/pg-tde-create-ext.sh /docker-entrypoint-initdb.d/pg-tde-create-ext.sh

expected/insert_update_delete.out

@@ -1,4 +1,16 @@
 CREATE EXTENSION pg_tde;
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+ pg_tde_add_key_provider_file 
+------------------------------
+                            1
+(1 row)
+
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+ pg_tde_set_master_key 
+-----------------------
+ 
+(1 row)
+
 CREATE TABLE albums (
     id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
     artist VARCHAR(256),

expected/vault_v2_test.out

@@ -1,5 +1,6 @@
 CREATE EXTENSION pg_tde;
-SELECT pg_tde_add_key_provider_vault_v2('vault-v2','ROOT_TOKEN','http://127.0.0.1:8200','secret',NULL);
+\getenv root_token ROOT_TOKEN
+SELECT pg_tde_add_key_provider_vault_v2('vault-v2',:'root_token','http://127.0.0.1:8200','secret',NULL);
  pg_tde_add_key_provider_vault_v2 
 ----------------------------------
                                 1

keyring-vault.json

@@ -1,6 +0,0 @@
-{
-        "provider": "vault-v2",
-        "token": "ROOT_TOKEN",
-        "url": "http://127.0.0.1:8200",
-        "mountPath": "secret"
-}

keyring.json

@@ -1,4 +0,0 @@
-{
-        "provider": "file",
-        "datafile": "/tmp/pgkeyring"
-}

meson.build

@@ -75,6 +75,7 @@ tests += {
       'multi_insert',
       'trigger_on_view',
       'insert_update_delete',
+      'vault_v2_test',
     ],
     'regress_args': ['--temp-config', files('pg_tde.conf')],
     'runningcheck': false,

pg_tde.conf

@@ -1,2 +1 @@
 shared_preload_libraries = 'pg_tde'
-pg_tde.keyringConfigFile = '/tmp/keyring.json'

sql/insert_update_delete.sql

@@ -1,5 +1,8 @@
 CREATE EXTENSION pg_tde;
 
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+
 CREATE TABLE albums (
     id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
     artist VARCHAR(256),

sql/vault_v2_test.sql

@@ -1,6 +1,7 @@
 CREATE EXTENSION pg_tde;
 
-SELECT pg_tde_add_key_provider_vault_v2('vault-v2','ROOT_TOKEN','http://127.0.0.1:8200','secret',NULL);
+\getenv root_token ROOT_TOKEN
+SELECT pg_tde_add_key_provider_vault_v2('vault-v2',:'root_token','http://127.0.0.1:8200','secret',NULL);
 SELECT pg_tde_set_master_key('vault-v2-master-key','vault-v2');
 
 CREATE TABLE test_enc(

src/include/keyring/keyring_config.h

@@ -14,8 +14,6 @@ enum KeyringProvider
 } ;
 
 extern enum KeyringProvider keyringProvider;
-extern char* keyringConfigFile;
-extern char* keyringKeyPrefix;
 
 void keyringRegisterVariables(void);
 

src/keyring/keyring_config.c

@@ -10,83 +10,11 @@
 
 #include "utils/guc.h"
 
-char* keyringConfigFile = "";
-char* keyringKeyPrefix = "";
 enum KeyringProvider keyringProvider = PROVIDER_UNKNOWN;
 
-static bool keyringCheckKeyPrefix(char **newval, void **extra, GucSource source)
-{
-	if(*newval == NULL || strlen(*newval) == 0)
-	{
-		return 1; // empty
-	}
-
-	if(strlen(*newval) > 32)
-	{
-		elog(ERROR, "The maximum length of pg_tde.keyringKeyPrefix is 32 characters.");
-		return 0;
-	}
-
-	return 1;
-}
-
-static bool keyringCheckConfigFile(char **newval, void **extra, GucSource source)
-{
-	if(*newval == NULL || strlen(*newval) == 0)
-	{
-		return 1; // empty
-	}
-
-	if(access(*newval, R_OK) != 0)
-	{
-		elog(ERROR, "The file referenced by pg_tde.keyringConfigFile doesn't exists, or is not readable to postgres");
-		return 0;
-	}
-
-	if(access(*newval, W_OK) == 0)
-	{
-		elog(WARNING, "The file referenced by pg_tde.keyringConfigFile is writable for the database process");
-	}
-
-	return 1;
-}
-
-static void keyringAssignConfigFile(const char *newval, void *extra)
-{
-	// TODO: make sure we only load the configuration once...
-	if(newval == NULL || strlen(newval) == 0)
-	{
-		//elog(WARNING, "pg_tde.keyringConfigFile is empty. Encryption features will not be available.");
-		return;
-	} 
-}
-
 void keyringRegisterVariables(void)
 {
-
-	DefineCustomStringVariable("pg_tde.keyringConfigFile", /* name */
-							"Location of the configuration file for the keyring", /* short_desc */
-							NULL,	/* long_desc */
-							&keyringConfigFile,	/* value address */
-							"",	/* boot value */
-							PGC_POSTMASTER, /* context */
-							0,	/* flags */
-							&keyringCheckConfigFile,	/* check_hook */
-							&keyringAssignConfigFile,	/* assign_hook */
-							NULL	/* show_hook */
-		);
-
-	DefineCustomStringVariable("pg_tde.keyringKeyPrefix", /* name */
-							"Location of the configuration file for the keyring", /* short_desc */
-							NULL,	/* long_desc */
-							&keyringKeyPrefix,	/* value address */
-							"",	/* boot value */
-							PGC_POSTMASTER, /* context */
-							0,	/* flags */
-							&keyringCheckKeyPrefix,	/* check_hook */
-							NULL,	/* assign_hook */
-							NULL	/* show_hook */
-		);
+	// nop for now
 }
 
 const char* keyringParseStringParam(json_object* object)

src/keyring/keyring_file.c

@@ -60,6 +60,7 @@ get_key_by_name(GenericKeyring* keyring, const char* key_name, bool throw_error,
 		if (bytes_read == 0 )
 		{
 			pfree(key);
+			*return_code = KEYRING_CODE_RESOURCE_NOT_AVAILABLE;
 			return NULL;
 		}
 		if (bytes_read != sizeof(keyInfo))

src/keyring/keyring_vault.c

@@ -224,6 +224,12 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, bool throw_error,
 		goto cleanup;
 	}
 
+	if (httpCode == 404)
+	{
+		*return_code = KEYRING_CODE_RESOURCE_NOT_AVAILABLE;
+		goto cleanup;
+	}
+
 	if (httpCode / 100 != 2)
 	{
 		*return_code = KEYRING_CODE_INVALID_RESPONSE;
@@ -292,7 +298,7 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, bool throw_error,
 	key = palloc(sizeof(keyInfo));
 	key->data.len = pg_b64_decode(response_key, strlen(response_key), (char *)key->data.data, MAX_KEY_DATA_SIZE);
 
-	if (key->data.len != MAX_KEY_DATA_SIZE)
+	if (key->data.len > MAX_KEY_DATA_SIZE)
 	{
 		*return_code = KEYRING_CODE_INVALID_KEY_SIZE;
 		ereport(throw_error ? ERROR : WARNING,

t/001_basic.pl

@@ -39,12 +39,6 @@ ok($rt_value == 3, "Failing query");
 PGTDE::append_to_file("-- server restart");
 $node->stop();
 
-# UPDATE postgresql.conf to include/load pg_tde library
-open $conf, '>>', "$pgdata/postgresql.conf";
-#print $conf "pg_tde.keyringConfigFile = '/tmp/keyring.json'\n";
-print $conf "pg_tde.keyringKeyPrefix = 'this-is-a-prefix'\n";
-close $conf;
-
 $rt_value = $node->start();
 ok($rt_value == 1, "Restart Server");
 

t/results/001_basic.out

@@ -0,0 +1,13 @@
+CREATE EXTENSION pg_tde;
+-- server restart
+CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING pg_tde;
+INSERT INTO test_enc (k) VALUES (5),(6);
+SELECT * FROM test_enc ORDER BY id ASC;
+1|5
+2|6
+-- server restart
+SELECT * FROM test_enc ORDER BY id ASC;
+1|5
+2|6
+DROP TABLE test_enc;
+DROP EXTENSION pg_tde;