Remove 'percona_tde.pg_tde_key_provider' user catalog (#230)

* Remove 'percona_tde.pg_tde_key_provider' user catalog and introduce a provider info file for key providers

This commit removes the 'percona_tde.pg_tde_key_provider' user catalog and
replaces it with a provider info file to save key providers.
This change ensures that the key provider information can be accessed
during recovery, the user catalogs cannot be relied upon in such scenarios.

The commit maintains the current API functions, so callers will not experience
any differences in functionality or usage after this change.

Additionally, the commit adjusts how the shared memory manager retrieves
information about the number of LWLocks required by the extension, optimizing
the process.

TODO: Implement xlog message for cleaning up the provider info file during
recovery operations to ensure consistency and avoid potential issues.

expected/keyprovider_dependency.out

@@ -23,11 +23,4 @@ SELECT pg_tde_set_principal_key('test-db-principal-key','mk-file');
  t
 (1 row)
 
--- Try dropping the in-use key provider
-DELETE FROM percona_tde.pg_tde_key_provider WHERE provider_name = 'mk-file'; -- Should fail
-ERROR:  Key provider "mk-file" cannot be deleted
-DETAIL:  The principal key for the database depends on this key provider.
--- Now delete the un-used  key provider
-DELETE FROM percona_tde.pg_tde_key_provider WHERE provider_name = 'free-file'; -- Should pass
-DELETE FROM percona_tde.pg_tde_key_provider WHERE provider_name = 'V2-vault'; -- Should pass
 DROP EXTENSION pg_tde;

pg_tde--1.0.sql

@@ -3,37 +3,16 @@
 -- complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "CREATE EXTENSION pg_tde" to load this file. \quit
 
--- pg_tde catalog tables
-CREATE SCHEMA percona_tde;
--- Note: The table is created using heap storage becasue we do not want this table
--- to be encrypted by pg_tde. This table is used to store key provider information
--- and we do not want to encrypt this table using pg_tde.
-CREATE TABLE percona_tde.pg_tde_key_provider(provider_id SERIAL,
-        keyring_type VARCHAR(10) CHECK (keyring_type IN ('file', 'vault-v2')),
-        provider_name VARCHAR(255) UNIQUE NOT NULL, options JSON, PRIMARY KEY(provider_id)) using heap;
-
--- If you want to add new provider types, you need to make appropriate changes
--- in include/catalog/tde_keyring.h and src/catalog/tde_keyring.c files.
-
-SELECT pg_catalog.pg_extension_config_dump('percona_tde.pg_tde_key_provider', '');
-
--- Trigger function to check principal key dependency on key provider row
-CREATE FUNCTION keyring_delete_dependency_check_trigger()
-RETURNS TRIGGER
+-- Key Provider Management
+CREATE FUNCTION pg_tde_add_key_provider_internal(provider_type VARCHAR(10), provider_name VARCHAR(128), options JSON)
+RETURNS INT
 AS 'MODULE_PATHNAME'
 LANGUAGE C;
 
-CREATE TRIGGER pg_tde_key_provider_delete_dependency_check_trigger
-BEFORE DELETE ON percona_tde.pg_tde_key_provider
-FOR EACH ROW
-EXECUTE FUNCTION keyring_delete_dependency_check_trigger();
-
--- Key Provider Management
-
 CREATE OR REPLACE FUNCTION pg_tde_add_key_provider(provider_type VARCHAR(10), provider_name VARCHAR(128), options JSON)
 RETURNS INT
 AS $$
-    INSERT INTO percona_tde.pg_tde_key_provider (keyring_type, provider_name, options) VALUES (provider_type, provider_name, options) RETURNING provider_id;
+    SELECT pg_tde_add_key_provider_internal(provider_type, provider_name, options);
 $$
 LANGUAGE SQL;
 
@@ -77,10 +56,6 @@ AS $$
 $$
 LANGUAGE SQL;
 
-CREATE FUNCTION pg_tde_get_keyprovider(provider_name text)
-RETURNS VOID
-AS 'MODULE_PATHNAME'
-LANGUAGE C;
 -- Table access method
 CREATE FUNCTION pg_tdeam_handler(internal)
 RETURNS table_am_handler

sql/keyprovider_dependency.sql

@@ -6,10 +6,4 @@ SELECT pg_tde_add_key_provider_vault_v2('V2-vault','vault-token','percona.com/va
 
 SELECT pg_tde_set_principal_key('test-db-principal-key','mk-file');
 
--- Try dropping the in-use key provider
-DELETE FROM percona_tde.pg_tde_key_provider WHERE provider_name = 'mk-file'; -- Should fail
--- Now delete the un-used  key provider
-DELETE FROM percona_tde.pg_tde_key_provider WHERE provider_name = 'free-file'; -- Should pass
-DELETE FROM percona_tde.pg_tde_key_provider WHERE provider_name = 'V2-vault'; -- Should pass
-
 DROP EXTENSION pg_tde;

src/access/pg_tde_tdemap.c

@@ -12,7 +12,6 @@
 
 #include "postgres.h"
 #include "access/pg_tde_tdemap.h"
-#include "catalog/pg_tablespace_d.h"
 #include "transam/pg_tde_xact_handler.h"
 #include "storage/fd.h"
 #include "utils/wait_event.h"
@@ -29,6 +28,7 @@
 #include "encryption/enc_aes.h"
 #include "encryption/enc_tde.h"
 #include "keyring/keyring_api.h"
+#include "common/pg_tde_utils.h"
 
 #include <openssl/rand.h>
 #include <openssl/err.h>
@@ -273,21 +273,13 @@ tde_decrypt_rel_key(TDEPrincipalKey *principal_key, RelKeyData *enc_rel_key_data
 inline void
 pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *map_path, char *keydata_path)
 {
-	char *db_path;
-
-	/* If this is a global space, than the call might be in a critial section
-	 * (during XLog write) so we can't do GetDatabasePath as it calls palloc()
-	 */
-	if (rlocator->spcOid == GLOBALTABLESPACE_OID)
-		db_path = "global";
-	else
-		db_path = GetDatabasePath(rlocator->dbOid, rlocator->spcOid);
-
+	char *db_path = pg_tde_get_tde_file_dir(rlocator->dbOid, rlocator->spcOid);
 
 	if (map_path)
 		join_path_components(map_path, db_path, PG_TDE_MAP_FILENAME);
 	if (keydata_path)
 		join_path_components(keydata_path, db_path, PG_TDE_KEYDATA_FILENAME);
+	pfree(db_path);
 }
 
 /*

src/catalog/tde_keyring.c

@@ -1,7 +1,7 @@
 /*-------------------------------------------------------------------------
  *
  * tde_keyring.c
- *      Deals with the tde keyring configuration catalog
+ *      Deals with the tde keyring configuration
  *      routines.
  *
  * IDENTIFICATION
@@ -21,18 +21,18 @@
 #include "utils/snapmgr.h"
 #include "utils/fmgroids.h"
 #include "common/pg_tde_utils.h"
+#include "common/pg_tde_shmem.h"
 #include "executor/spi.h"
+#include "miscadmin.h"
 #include "unistd.h"
 #include "utils/builtins.h"
+#include "pg_tde.h"
 
-PG_FUNCTION_INFO_V1(keyring_delete_dependency_check_trigger);
+PG_FUNCTION_INFO_V1(pg_tde_add_key_provider_internal);
+Datum pg_tde_add_key_provider_internal(PG_FUNCTION_ARGS);
 
-/* Must match the catalog table definition */
-#define PG_TDE_KEY_PROVIDER_ID_ATTRNUM 1
-#define PG_TDE_KEY_PROVIDER_TYPE_ATTRNUM 2
-#define PG_TDE_KEY_PROVIDER_NAME_ATTRNUM 3
-#define PG_TDE_KEY_PROVIDER_OPTIONS_ATTRNUM 4
 
+#define PG_TDE_KEYRING_FILENAME "pg_tde_keyrings"
 /*
  * These token must be exactly same as defined in
  * pg_tde_add_key_provider_vault_v2 SQL interface
@@ -49,11 +49,88 @@ PG_FUNCTION_INFO_V1(keyring_delete_dependency_check_trigger);
 #define FILE_KEYRING_PATH_KEY "path"
 #define FILE_KEYRING_TYPE_KEY "type"
 
+typedef enum ProviderScanType
+{
+	PROVIDER_SCAN_BY_NAME,
+	PROVIDER_SCAN_BY_ID,
+	PROVIDER_SCAN_BY_TYPE,
+	PROVIDER_SCAN_ALL
+} ProviderScanType;
+
+static List *scan_key_provider_file(ProviderScanType scanType, void *scanKey);
+
 static FileKeyring *load_file_keyring_provider_options(Datum keyring_options);
 static GenericKeyring *load_keyring_provider_options(ProviderType provider_type, Datum keyring_options);
 static VaultV2Keyring *load_vaultV2_keyring_provider_options(Datum keyring_options);
 static void debug_print_kerying(GenericKeyring *keyring);
-static GenericKeyring *load_keyring_provider_from_tuple(HeapTuple tuple, TupleDesc tupDesc);
+static char *get_keyring_infofile_path(char *resPath, Oid dbOid, Oid spcOid);
+static uint32 save_key_provider(KeyringProvideRecord *provider);
+static void key_provider_startup_cleanup(int tde_tbl_count, void *arg);
+
+static Size initialize_shared_state(void *start_address);
+static Size required_shared_mem_size(void);
+
+typedef struct TdeKeyProviderInfoSharedState
+{
+	LWLock *Locks;
+} TdeKeyProviderInfoSharedState;
+
+TdeKeyProviderInfoSharedState*	sharedPrincipalKeyState = NULL; /* Lives in shared state */
+
+static const TDEShmemSetupRoutine key_provider_info_shmem_routine = {
+	.init_shared_state = initialize_shared_state,
+	.init_dsa_area_objects = NULL,
+	.required_shared_mem_size = required_shared_mem_size,
+	.shmem_kill = NULL
+	};
+
+static Size
+required_shared_mem_size(void)
+{
+	return MAXALIGN(sizeof(TdeKeyProviderInfoSharedState));
+}
+
+static Size
+initialize_shared_state(void *start_address)
+{
+	sharedPrincipalKeyState = (TdeKeyProviderInfoSharedState *)start_address;
+	sharedPrincipalKeyState->Locks = GetLWLocks();
+	return sizeof(TdeKeyProviderInfoSharedState);
+}
+
+static inline LWLock *
+tde_provider_info_lock(void)
+{
+	Assert(sharedPrincipalKeyState);
+	return &sharedPrincipalKeyState->Locks[TDE_LWLOCK_PI_FILES];
+}
+
+void InitializeKeyProviderInfo(void)
+{
+	ereport(LOG, (errmsg("initializing TDE key provider info")));
+	RegisterShmemRequest(&key_provider_info_shmem_routine);
+	on_ext_install(key_provider_startup_cleanup, NULL);
+}
+static void
+key_provider_startup_cleanup(int tde_tbl_count, void *arg)
+{
+
+	if (tde_tbl_count > 0)
+	{
+		ereport(WARNING,
+				(errmsg("failed to perform initialization. database already has %d TDE tables", tde_tbl_count)));
+		return;
+	}
+	cleanup_key_provider_info(MyDatabaseId, MyDatabaseTableSpace);
+
+	/* TODO: XLog the key cleanup */
+	// XLogPrincipalKeyCleanup xlrec;
+	// xlrec.databaseId = MyDatabaseId;
+	// xlrec.tablespaceId = MyDatabaseTableSpace;
+	// XLogBeginInsert();
+	// XLogRegisterData((char *)&xlrec, sizeof(TDEPrincipalKeyInfo));
+	// XLogInsert(RM_TDERMGR_ID, XLOG_TDE_CLEAN_PRINCIPAL_KEY);
+}
 
 ProviderType
 get_keyring_provider_from_typename(char *provider_type)
@@ -69,36 +146,19 @@ get_keyring_provider_from_typename(char *provider_type)
 }
 
 static GenericKeyring *
-load_keyring_provider_from_tuple(HeapTuple tuple, TupleDesc tupDesc)
+load_keyring_provider_from_record(KeyringProvideRecord* provider)
 {
-	Datum datum;
 	Datum option_datum;
-	bool isnull;
-	char *keyring_name;
-	char *keyring_type;
-	int provider_id;
-	ProviderType provider_type = UNKNOWN_KEY_PROVIDER;
 	GenericKeyring *keyring = NULL;
 
-	datum = heap_getattr(tuple, PG_TDE_KEY_PROVIDER_ID_ATTRNUM, tupDesc, &isnull);
-	provider_id = DatumGetInt32(datum);
-
-	datum = heap_getattr(tuple, PG_TDE_KEY_PROVIDER_TYPE_ATTRNUM, tupDesc, &isnull);
-	keyring_type = TextDatumGetCString(datum);
-
-	datum = heap_getattr(tuple, PG_TDE_KEY_PROVIDER_NAME_ATTRNUM, tupDesc, &isnull);
-	keyring_name = TextDatumGetCString(datum);
+	option_datum = CStringGetTextDatum(provider->options);
 
-	option_datum = heap_getattr(tuple, PG_TDE_KEY_PROVIDER_OPTIONS_ATTRNUM, tupDesc, &isnull);
-
-	provider_type = get_keyring_provider_from_typename(keyring_type);
-
-	keyring = load_keyring_provider_options(provider_type, option_datum);
+	keyring = load_keyring_provider_options(provider->provider_type, option_datum);
 	if (keyring)
 	{
-		strncpy(keyring->provider_name, keyring_name, sizeof(keyring->provider_name));
-		keyring->key_id = provider_id;
-		keyring->type = provider_type;
+		keyring->key_id = provider->provider_id;
+		strncpy(keyring->provider_name, provider->provider_name, sizeof(keyring->provider_name));
+		keyring->type = provider->provider_type;
 		debug_print_kerying(keyring);
 	}
 	return keyring;
@@ -107,103 +167,39 @@ load_keyring_provider_from_tuple(HeapTuple tuple, TupleDesc tupDesc)
 List *
 GetAllKeyringProviders(void)
 {
-	HeapTuple tuple;
-	TupleDesc tupDesc;
-	List *keyring_list = NIL;
-
-	Oid pg_tde_schema_oid = LookupNamespaceNoError(PG_TDE_NAMESPACE_NAME);
-	Oid kp_table_oid = get_relname_relid(PG_TDE_KEY_PROVIDER_CAT_NAME, pg_tde_schema_oid);
-	Relation kp_table_relation = relation_open(kp_table_oid, AccessShareLock);
-	TableScanDesc scan;
-
-	tupDesc = kp_table_relation->rd_att;
-
-	scan = heap_beginscan(kp_table_relation, GetLatestSnapshot(), 0, NULL, NULL, 0);
-
-	while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL)
-	{
-		GenericKeyring *keyring = load_keyring_provider_from_tuple(tuple, tupDesc);
-		if (keyring)
-		{
-			keyring_list = lappend(keyring_list, keyring);
-		}
-	}
-	heap_endscan(scan);
-	relation_close(kp_table_relation, AccessShareLock);
-
-	return keyring_list;
+	return scan_key_provider_file(PROVIDER_SCAN_ALL, NULL);
 }
 
 GenericKeyring *
 GetKeyProviderByName(const char *provider_name)
 {
-	HeapTuple tuple;
-	TupleDesc tupDesc;
-	TableScanDesc scan;
-	ScanKeyData scanKey[1];
 	GenericKeyring *keyring = NULL;
-	Oid pg_tde_schema_oid = LookupNamespaceNoError(PG_TDE_NAMESPACE_NAME);
-	Oid kp_table_oid = get_relname_relid(PG_TDE_KEY_PROVIDER_CAT_NAME, pg_tde_schema_oid);
-	Relation kp_table_relation = relation_open(kp_table_oid, AccessShareLock);
-
-	/* Set up a scan key to fetch only required record. */
-	ScanKeyInit(scanKey,
-				(AttrNumber)PG_TDE_KEY_PROVIDER_NAME_ATTRNUM,
-				BTEqualStrategyNumber, F_TEXTEQ,
-				CStringGetTextDatum(provider_name));
-
-	tupDesc = kp_table_relation->rd_att;
-	/* Begin the scan with the filter condition */
-	scan = heap_beginscan(kp_table_relation, GetLatestSnapshot(), 1, scanKey, NULL, 0);
-
-	while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL)
+	List *providers = scan_key_provider_file(PROVIDER_SCAN_BY_NAME, (void*)provider_name);
+	if (providers != NIL)
 	{
-		keyring = load_keyring_provider_from_tuple(tuple, tupDesc);
-		break; /* We expect only one record */
+		keyring = (GenericKeyring *)linitial(providers);
+		list_free(providers);
+	}
+	else
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("key provider \"%s\" does not exists", provider_name),
+				 errhint("Use pg_tde_add_key_provider interface to create the key provider")));
 	}
-	heap_endscan(scan);
-	relation_close(kp_table_relation, AccessShareLock);
-
-    if (keyring == NULL)
-    {
-        ereport(ERROR,
-                (errmsg("Key provider \"%s\" does not exists", provider_name),
-                 errhint("Use create_key_provider interface to create the key provider")));
-    }
-
 	return keyring;
 }
 
 GenericKeyring *
 GetKeyProviderByID(int provider_id)
 {
-	HeapTuple tuple;
-	TupleDesc tupDesc;
-	TableScanDesc scan;
-	ScanKeyData scanKey[1];
 	GenericKeyring *keyring = NULL;
-	Oid pg_tde_schema_oid = LookupNamespaceNoError(PG_TDE_NAMESPACE_NAME);
-	Oid kp_table_oid = get_relname_relid(PG_TDE_KEY_PROVIDER_CAT_NAME, pg_tde_schema_oid);
-	Relation kp_table_relation = relation_open(kp_table_oid, AccessShareLock);
-
-	/* Set up a scan key to fetch only required record. */
-	ScanKeyInit(scanKey,
-				(AttrNumber)PG_TDE_KEY_PROVIDER_ID_ATTRNUM,
-				BTEqualStrategyNumber, F_INT4EQ,
-				Int32GetDatum(provider_id));
-
-	tupDesc = kp_table_relation->rd_att;
-	/* Begin the scan with the filter condition */
-	scan = heap_beginscan(kp_table_relation, GetLatestSnapshot(), 1, scanKey, NULL, 0);
-
-	while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL)
+	List *providers = scan_key_provider_file(PROVIDER_SCAN_BY_ID, &provider_id);
+	if (providers != NIL)
 	{
-		keyring = load_keyring_provider_from_tuple(tuple, tupDesc);
-		break; /* We only expect 1 record */
+		keyring = (GenericKeyring *)linitial(providers);
+		list_free(providers);
 	}
-	heap_endscan(scan);
-	relation_close(kp_table_relation, AccessShareLock);
-
 	return keyring;
 }
 
@@ -232,7 +228,9 @@ load_file_keyring_provider_options(Datum keyring_options)
 	
 	if(file_path == NULL)
 	{
-		/* TODO: report error */
+		ereport(DEBUG2,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("file path is missing in the keyring options")));
 		return NULL;
 	}
 
@@ -267,99 +265,224 @@ load_vaultV2_keyring_provider_options(Datum keyring_options)
 static void
 debug_print_kerying(GenericKeyring *keyring)
 {
-	elog(DEBUG2, "Keyring type: %d", keyring->type);
-	elog(DEBUG2, "Keyring name: %s", keyring->provider_name);
-	elog(DEBUG2, "Keyring id: %d", keyring->key_id);
+	int debug_level = DEBUG2;
+	elog(debug_level, "Keyring type: %d", keyring->type);
+	elog(debug_level, "Keyring name: %s", keyring->provider_name);
+	elog(debug_level, "Keyring id: %d", keyring->key_id);
 	switch (keyring->type)
 	{
 	case FILE_KEY_PROVIDER:
-		elog(DEBUG2, "File Keyring Path: %s", ((FileKeyring *)keyring)->file_name);
+		elog(debug_level, "File Keyring Path: %s", ((FileKeyring *)keyring)->file_name);
 		break;
 	case VAULT_V2_KEY_PROVIDER:
-		elog(DEBUG2, "Vault Keyring Token: %s", ((VaultV2Keyring *)keyring)->vault_token);
-		elog(DEBUG2, "Vault Keyring URL: %s", ((VaultV2Keyring *)keyring)->vault_url);
-		elog(DEBUG2, "Vault Keyring Mount Path: %s", ((VaultV2Keyring *)keyring)->vault_mount_path);
-		elog(DEBUG2, "Vault Keyring CA Path: %s", ((VaultV2Keyring *)keyring)->vault_ca_path);
+		elog(debug_level, "Vault Keyring Token: %s", ((VaultV2Keyring *)keyring)->vault_token);
+		elog(debug_level, "Vault Keyring URL: %s", ((VaultV2Keyring *)keyring)->vault_url);
+		elog(debug_level, "Vault Keyring Mount Path: %s", ((VaultV2Keyring *)keyring)->vault_mount_path);
+		elog(debug_level, "Vault Keyring CA Path: %s", ((VaultV2Keyring *)keyring)->vault_ca_path);
 		break;
 	case UNKNOWN_KEY_PROVIDER:
-		elog(DEBUG2, "Unknown Keyring ");
+		elog(debug_level, "Unknown Keyring ");
 		break;
 	}
 }
 
 /*
- * Trigger function on keyring catalog to ensure the keyring
- * used by the principal key should not get deleted.
- */
-Datum
-keyring_delete_dependency_check_trigger(PG_FUNCTION_ARGS)
+ * Fetch the next key provider from the file and update the curr_pos
+*/
+static bool
+fetch_next_key_provider(int fd, off_t* curr_pos, KeyringProvideRecord *provider)
 {
-	TriggerData *trig_data = (TriggerData *)fcinfo->context;
-	Oid principal_key_keyring_id;
+	off_t bytes_read = 0;
+
+	Assert(provider != NULL);
+	Assert(fd >= 0);
 
-	if (!CALLED_AS_TRIGGER(fcinfo))
+	bytes_read = pg_pread(fd, provider, sizeof(KeyringProvideRecord), *curr_pos);
+	*curr_pos += bytes_read;
+
+	if (bytes_read == 0)
+		return false;
+	if (bytes_read != sizeof(KeyringProvideRecord))
 	{
+		close(fd);
+		/* Corrupt file */
 		ereport(ERROR,
-				(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
-				 errmsg("keyring dependency check trigger: not fired by trigger manager")));
+				(errcode_for_file_access(),
+					errmsg("key provider info file is corrupted: %m"),
+					errdetail("invalid key provider record size %lld expected %lu", bytes_read, sizeof(KeyringProvideRecord) )));
 	}
+	return true;
+}
+
+/*
+* Save the key provider info to the file
+*/
+static uint32
+save_key_provider(KeyringProvideRecord *provider)
+{
+	off_t bytes_written = 0;
+	off_t curr_pos = 0;
+	int fd;
+	int max_provider_id = 0;
+	char kp_info_path[MAXPGPATH] = {0};
+	KeyringProvideRecord existing_provider;
+
+	Assert(provider != NULL);
 
-	if (!TRIGGER_FIRED_BEFORE(trig_data->tg_event))
+	get_keyring_infofile_path(kp_info_path, MyDatabaseId, MyDatabaseTableSpace);
+
+	LWLockAcquire(tde_provider_info_lock(), LW_EXCLUSIVE);
+
+	fd = BasicOpenFile(kp_info_path, O_CREAT | O_RDWR | PG_BINARY);
+	if (fd < 0)
+	{
+		LWLockRelease(tde_provider_info_lock());
+		ereport(ERROR,
+			(errcode_for_file_access(),
+				errmsg("could not open tde file \"%s\": %m", kp_info_path)));
+	}
+
+	/* we also need to verify the name conflict and generate the next provider ID */
+	while (fetch_next_key_provider(fd, &curr_pos, &existing_provider))
 	{
+		if (strcmp(existing_provider.provider_name, provider->provider_name) == 0)
+		{
+			close(fd);
+			LWLockRelease(tde_provider_info_lock());
+			ereport(ERROR,
+				(errcode(ERRCODE_DUPLICATE_OBJECT),
+					errmsg("key provider \"%s\" already exists", provider->provider_name)));
+		}
+		if (max_provider_id < existing_provider.provider_id)
+			max_provider_id = existing_provider.provider_id;
+	}
+	provider->provider_id = max_provider_id + 1;
+	/*
+	 * All good, Just add a new provider
+	 * Write key to the end of file
+	 */
+	curr_pos = lseek(fd, 0, SEEK_END);
+	bytes_written = pg_pwrite(fd, provider, sizeof(KeyringProvideRecord), curr_pos);
+	if (bytes_written != sizeof(KeyringProvideRecord))
+	{
+		close(fd);
+		LWLockRelease(tde_provider_info_lock());
 		ereport(ERROR,
-				(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
-				 errmsg("keyring dependency check trigger: trigger should be fired before delete")));
+			(errcode_for_file_access(),
+				errmsg("key provider info file \"%s\" can't be written: %m",
+						kp_info_path)));
 	}
-	principal_key_keyring_id = GetPrincipalKeyProviderId();
-	if (principal_key_keyring_id == InvalidOid)
+	if (pg_fsync(fd) != 0)
 	{
-		/* No principal key set. We are good to delete anything */
-		return PointerGetDatum(trig_data->tg_trigtuple);
+		close(fd);
+		LWLockRelease(tde_provider_info_lock());
+		ereport(ERROR,
+			(errcode_for_file_access(),
+				errmsg("could not fsync file \"%s\": %m",
+						kp_info_path)));
 	}
+	close(fd);
+	LWLockRelease(tde_provider_info_lock());
+	return provider->provider_id;
+}
 
-	if (TRIGGER_FIRED_BY_DELETE(trig_data->tg_event))
+/*
+ * Scan the key provider info file and can also apply filter based on scanType
+ */
+static List *
+scan_key_provider_file(ProviderScanType scanType, void* scanKey)
+{
+	off_t curr_pos = 0;
+	int fd;
+	char kp_info_path[MAXPGPATH] = {0};
+	KeyringProvideRecord provider;
+	List *providers_list = NIL;
+
+	if (scanType != PROVIDER_SCAN_ALL)
+		Assert(scanKey != NULL);
+
+	get_keyring_infofile_path(kp_info_path, MyDatabaseId, MyDatabaseTableSpace);
+
+	LWLockAcquire(tde_provider_info_lock(), LW_SHARED);
+
+	fd = BasicOpenFile(kp_info_path, PG_BINARY);
+	if (fd < 0)
+	{
+		LWLockRelease(tde_provider_info_lock());
+		ereport(DEBUG2,
+			(errcode_for_file_access(),
+				errmsg("could not open tde file \"%s\": %m", kp_info_path)));
+		return NIL;
+	}
+	while (fetch_next_key_provider(fd, &curr_pos, &provider))
 	{
-		HeapTuple oldtuple = trig_data->tg_trigtuple;
+		bool match = false;
+		ereport(DEBUG2,
+			(errmsg("read key provider ID=%d %s", provider.provider_id, provider.provider_name)));
+
+		if (scanType == PROVIDER_SCAN_BY_NAME)
+		{
+			if (strcasecmp(provider.provider_name, (char*)scanKey) == 0)
+				match = true;
+		}
+		else if (scanType == PROVIDER_SCAN_BY_ID)
+		{
+			if (provider.provider_id == *(int *)scanKey)
+				match = true;
+		}
+		else if (scanType == PROVIDER_SCAN_BY_TYPE)
+		{
+			if (provider.provider_type == *(ProviderType*)scanKey)
+				match = true;
+		}
+		else if (scanType == PROVIDER_SCAN_ALL)
+			match = true;
 
-		if (oldtuple != NULL && SPI_connect() == SPI_OK_CONNECT)
+		if (match)
 		{
-			Datum datum;
-			bool isnull;
-			Oid provider_id;
-			datum = heap_getattr(oldtuple, PG_TDE_KEY_PROVIDER_ID_ATTRNUM, trig_data->tg_relation->rd_att, &isnull);
-			provider_id = DatumGetInt32(datum);
-			if (provider_id == principal_key_keyring_id)
+			GenericKeyring *keyring = load_keyring_provider_from_record(&provider);
+			if (keyring)
 			{
-				char *keyring_name;
-				datum = heap_getattr(oldtuple, PG_TDE_KEY_PROVIDER_NAME_ATTRNUM, trig_data->tg_relation->rd_att, &isnull);
-				keyring_name = TextDatumGetCString(datum);
-
-				ereport(ERROR,
-						(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
-						 errmsg("Key provider \"%s\" cannot be deleted", keyring_name),
-						 errdetail("The principal key for the database depends on this key provider.")));
-				SPI_finish();
-				trig_data->tg_trigtuple = NULL;
-				return PointerGetDatum(NULL);
+				providers_list = lappend(providers_list, keyring);
 			}
-			SPI_finish();
 		}
 	}
-	else
-		ereport(ERROR,
-				(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
-				 errmsg("keyring_delete_dependency_check_trigger: unsupported event type")));
+	close(fd);
+	LWLockRelease(tde_provider_info_lock());
+	return providers_list;
+}
+
+void
+cleanup_key_provider_info(Oid databaseId, Oid tablespaceId)
+{
+	/* Remove the key provider info fileß */
+	char kp_info_path[MAXPGPATH] = {0};
 
-	/* Indicate that the operation should proceed */
-	return PointerGetDatum(trig_data->tg_trigtuple);
+	get_keyring_infofile_path(kp_info_path, MyDatabaseId, MyDatabaseTableSpace);
+	PathNameDeleteTemporaryFile(kp_info_path, false);
 }
 
-/* Testing function */
-PG_FUNCTION_INFO_V1(pg_tde_get_keyprovider);
-Datum pg_tde_get_keyprovider(PG_FUNCTION_ARGS);
+static char*
+get_keyring_infofile_path(char* resPath, Oid dbOid, Oid spcOid)
+{
+	char *db_path = pg_tde_get_tde_file_dir(dbOid, spcOid);
+	Assert(db_path != NULL);
+	join_path_components(resPath, db_path, PG_TDE_KEYRING_FILENAME);
+	pfree(db_path);
+	return resPath;
+}
 
-Datum pg_tde_get_keyprovider(PG_FUNCTION_ARGS)
+Datum
+pg_tde_add_key_provider_internal(PG_FUNCTION_ARGS)
 {
-	GetAllKeyringProviders();
-	PG_RETURN_NULL();
+	char *provider_type = text_to_cstring(PG_GETARG_TEXT_PP(0));
+	char *provider_name = text_to_cstring(PG_GETARG_TEXT_PP(1));
+	char *options = text_to_cstring(PG_GETARG_TEXT_PP(2));
+	KeyringProvideRecord provider;
+
+	strncpy(provider.options, options, sizeof(provider.options));
+	strncpy(provider.provider_name, provider_name, sizeof(provider.provider_name));
+	provider.provider_type = get_keyring_provider_from_typename(provider_type);
+	save_key_provider(&provider);
+	PG_RETURN_INT32(provider.provider_id);
 }

src/catalog/tde_principal_key.c

@@ -65,7 +65,6 @@ static Size initialize_shared_state(void *start_address);
 static void initialize_objects_in_dsa_area(dsa_area *dsa, void *raw_dsa_area);
 static Size cache_area_size(void);
 static Size required_shared_mem_size(void);
-static int  required_locks_count(void);
 static void shared_memory_shutdown(int code, Datum arg);
 static void principal_key_startup_cleanup(int tde_tbl_count, void *arg);
 static void clear_principal_key_cache(Oid databaseId) ;
@@ -77,7 +76,6 @@ static const TDEShmemSetupRoutine principal_key_info_shmem_routine = {
     .init_shared_state = initialize_shared_state,
     .init_dsa_area_objects = initialize_objects_in_dsa_area,
     .required_shared_mem_size = required_shared_mem_size,
-    .required_locks_count = required_locks_count,
     .shmem_kill = shared_memory_shutdown
     };
 
@@ -104,12 +102,6 @@ tde_lwlock_mk_cache(void)
     return &principalKeyLocalState.sharedPrincipalKeyState->Locks[TDE_LWLOCK_MK_CACHE];
 }
 
-static int
-required_locks_count(void)
-{
-    return TDE_LWLOCK_COUNT;
-}
-
 static Size
 cache_area_size(void)
 {
@@ -137,7 +129,7 @@ initialize_shared_state(void *start_address)
     principalKeyLocalState.dsa = NULL;
     principalKeyLocalState.sharedHash = NULL;
 
-    sharedState->Locks = GetNewLWLock();
+    sharedState->Locks = GetLWLocks();
     principalKeyLocalState.sharedPrincipalKeyState = sharedState;
     return sizeof(TdePrincipalKeySharedState);
 }

src/common/pg_tde_shmem.c

@@ -59,15 +59,7 @@ Size TdeRequiredSharedMemorySize(void)
 
 int TdeRequiredLocksCount(void)
 {
-	int count = 0;
-	ListCell *lc;
-	foreach (lc, registeredShmemRequests)
-	{
-		TDEShmemSetupRoutine *routine = (TDEShmemSetupRoutine *)lfirst(lc);
-		if (routine->required_locks_count)
-			count += routine->required_locks_count();
-	}
-	return count;
+	return TDE_LWLOCK_COUNT;
 }
 
 void TdeShmemInit(void)
@@ -147,13 +139,10 @@ tde_shmem_shutdown(int code, Datum arg)
 }
 
 /*
- * Returns a lock from registered named tranch.
- * You must already have indicated number of required locks
- * through required_locks_count callback before requesting
- * the lock from this function.
+ * Returns a locks array from registered named tranch.
  */
-LWLock*
-GetNewLWLock(void)
+LWLock *
+GetLWLocks(void)
 {
 	return &(GetNamedLWLockTranche(TDE_TRANCHE_NAME))->lock;
 }

src/common/pg_tde_utils.c

@@ -17,6 +17,7 @@
 #include "access/htup_details.h"
 #include "catalog/pg_class.h"
 #include "catalog/pg_am.h"
+#include "catalog/pg_tablespace_d.h"
 #include "utils/fmgroids.h"
 #include "utils/rel.h"
 #include "utils/snapmgr.h"
@@ -215,3 +216,15 @@ extract_json_option_value(Datum top_json, const char* field_name)
 		return NULL;
 	}
 }
+
+/* returns the palloc'd string */
+char *
+pg_tde_get_tde_file_dir(Oid dbOid, Oid spcOid)
+{
+	/* `dbOid` is set to a value for the XLog keys caching but GetDatabasePath() 
+	 * expects it (`dbOid`) to be `0` if this is a global space.
+	 */
+	if (spcOid == GLOBALTABLESPACE_OID)
+		return pstrdup("global");
+	return GetDatabasePath(dbOid, spcOid);
+}

src/include/catalog/tde_keyring.h

@@ -24,6 +24,7 @@
 
 #define MAX_PROVIDER_NAME_LEN 128 /* pg_tde_key_provider's provider_name size*/
 #define MAX_VAULT_V2_KEY_LEN 128 /* From hashi corp docs */
+#define MAX_KEYRING_OPTION_LEN	1024
 typedef enum ProviderType
 {
 	UNKNOWN_KEY_PROVIDER,
@@ -60,9 +61,19 @@ typedef union KeyringProviders
 	VaultV2Keyring vault;
 } KeyringProviders;
 
+/* This record goes into key provider info file */
+typedef struct KeyringProvideRecord
+{
+	int provider_id;
+	char provider_name[MAX_PROVIDER_NAME_LEN];
+	char options[MAX_KEYRING_OPTION_LEN];
+	ProviderType provider_type;
+} KeyringProvideRecord;
+
 extern List *GetAllKeyringProviders(void);
 extern GenericKeyring *GetKeyProviderByName(const char *provider_name);
 extern GenericKeyring *GetKeyProviderByID(int provider_id);
 extern ProviderType get_keyring_provider_from_typename(char *provider_type);
-
+extern void cleanup_key_provider_info(Oid databaseId, Oid tablespaceId);
+extern void InitializeKeyProviderInfo(void);
 #endif /*TDE_KEYRING_H*/

src/include/common/pg_tde_shmem.h

@@ -20,6 +20,7 @@ typedef enum
 {
     TDE_LWLOCK_MK_CACHE,
     TDE_LWLOCK_MK_FILES,
+    TDE_LWLOCK_PI_FILES,
 
     /* Must be the last entry in the enum */
     TDE_LWLOCK_COUNT
@@ -43,10 +44,6 @@ typedef struct TDEShmemSetupRoutine
      * The callback must return the size of the shared memory acquired.
      */
     Size (*required_shared_mem_size)(void);
-    /*
-     * Optional callback to return the number of LW locks required.
-     */
-    int (*required_locks_count)(void);
     /*
      * Gets called after all shared memory structures are initialized and
      * here you can create shared memory hash tables or any other shared
@@ -60,6 +57,6 @@ extern void RegisterShmemRequest(const TDEShmemSetupRoutine *routine);
 extern void TdeShmemInit(void);
 extern Size TdeRequiredSharedMemorySize(void);
 extern int TdeRequiredLocksCount(void);
-extern LWLock * GetNewLWLock(void);
+extern LWLock *GetLWLocks(void);
 
 #endif /*PG_TDE_SHMEM_H*/

src/include/common/pg_tde_utils.h

@@ -10,6 +10,7 @@
 
 #include "postgres.h"
 #include "nodes/pg_list.h"
+#include "common/relpath.h"
 
 extern Oid get_tde_table_am_oid(void);
 extern Oid get_tde2_table_am_oid(void);
@@ -19,4 +20,5 @@ extern int get_tde_tables_count(void);
 extern const char *extract_json_cstr(Datum json, const char* field_name);
 const char *extract_json_option_value(Datum top_json, const char* field_name);
 
+extern char *pg_tde_get_tde_file_dir(Oid dbOid, Oid spcOid);
 #endif /*PG_TDE_UTILS_H*/

src/pg_tde.c

@@ -103,6 +103,7 @@ _PG_init(void)
 
 	keyringRegisterVariables();
 	InitializePrincipalKeyInfo();
+	InitializeKeyProviderInfo();
 #ifdef PERCONA_FORK
 	XLogInitGUC();
 	TDEGlCatInitGUC();