From cbec8ceb5232382cd81bb24c169cc7f7c069b316 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Mon, 15 Apr 2024 19:38:36 +0100 Subject: [PATCH 01/14] Temporary build fix with PG17 development branch Because the current smgr patchset is based on the PG17 branch, smgr development needs to be based on that. But our current copied PG16 code doesn't compile with PG17. This commit doesn't try to make the pg_tde AM work with PG17, only "comments out" with conditional commands parts that do not compile. PG16 builds work as before, without any changes. PG17 at least compiles with missing code, probably doesn't really work. --- src/access/pg_tde_prune.c | 12 +++++++----- src/access/pg_tde_vacuumlazy.c | 7 +++++-- src/access/pg_tdeam.c | 21 ++++++++++++++------- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/access/pg_tde_prune.c b/src/access/pg_tde_prune.c index 73a4dcea225..335edecb56b 100644 --- a/src/access/pg_tde_prune.c +++ b/src/access/pg_tde_prune.c @@ -127,6 +127,7 @@ pg_tde_page_prune_opt(Relation relation, Buffer buffer) if (RecoveryInProgress()) return; +#if PG_VERSION_NUM < 170000 /* * XXX: Magic to keep old_snapshot_threshold tests appear "working". They * currently are broken, and discussion of what to do about them is @@ -135,7 +136,7 @@ pg_tde_page_prune_opt(Relation relation, Buffer buffer) */ if (old_snapshot_threshold == 0) SnapshotTooOldMagicForTest(); - +#endif /* * First check whether there's any chance there's something to prune, * determining the appropriate horizon is a waste if there's no prune_xid @@ -166,14 +167,14 @@ pg_tde_page_prune_opt(Relation relation, Buffer buffer) if (!GlobalVisTestIsRemovableXid(vistest, prune_xid)) { - if (!OldSnapshotThresholdActive()) +#if PG_VERSION_NUM < 170000 + if ( !OldSnapshotThresholdActive()) return; - if (!TransactionIdLimitedForOldSnapshots(GlobalVisTestNonRemovableHorizon(vistest), relation, &limited_xmin, &limited_ts)) return; - +#endif if (!TransactionIdPrecedes(prune_xid, limited_xmin)) return; } @@ -539,6 +540,7 @@ heap_prune_satisfies_vacuum(PruneState *prstate, HeapTuple tup, Buffer buffer) */ if (GlobalVisTestIsRemovableXid(prstate->vistest, dead_after)) res = HEAPTUPLE_DEAD; +#if PG_VERSION_NUM < 170000 else if (OldSnapshotThresholdActive()) { /* haven't determined limited horizon yet, requests */ @@ -566,7 +568,7 @@ heap_prune_satisfies_vacuum(PruneState *prstate, HeapTuple tup, Buffer buffer) res = HEAPTUPLE_DEAD; } } - +#endif return res; } diff --git a/src/access/pg_tde_vacuumlazy.c b/src/access/pg_tde_vacuumlazy.c index bd7eca4e928..acf894f60c5 100644 --- a/src/access/pg_tde_vacuumlazy.c +++ b/src/access/pg_tde_vacuumlazy.c @@ -2828,8 +2828,11 @@ should_attempt_truncation(LVRelState *vacrel) { BlockNumber possibly_freeable; - if (!vacrel->do_rel_truncate || VacuumFailsafeActive || - old_snapshot_threshold >= 0) + if (!vacrel->do_rel_truncate || VacuumFailsafeActive +#if PG_VERSION_NUM < 170000 + || old_snapshot_threshold >= 0 +#endif + ) return false; possibly_freeable = vacrel->rel_pages - vacrel->nonempty_pages; diff --git a/src/access/pg_tdeam.c b/src/access/pg_tdeam.c index 0c168202912..7167613f85e 100644 --- a/src/access/pg_tdeam.c +++ b/src/access/pg_tdeam.c @@ -430,7 +430,9 @@ pg_tde_getpage(TableScanDesc sscan, BlockNumber block) LockBuffer(buffer, BUFFER_LOCK_SHARE); page = BufferGetPage(buffer); +#if PG_VERSION_NUM < 170000 TestForOldSnapshot(snapshot, scan->rs_base.rs_rd, page); +#endif lines = PageGetMaxOffsetNumber(page); ntup = 0; @@ -569,9 +571,9 @@ pg_tde_gettup_start_page(HeapScanDesc scan, ScanDirection dir, int *linesleft, /* Caller is responsible for ensuring buffer is locked if needed */ page = BufferGetPage(scan->rs_cbuf); - +#if PG_VERSION_NUM < 170000 TestForOldSnapshot(scan->rs_base.rs_snapshot, scan->rs_base.rs_rd, page); - +#endif *linesleft = PageGetMaxOffsetNumber(page) - FirstOffsetNumber + 1; if (ScanDirectionIsForward(dir)) @@ -602,9 +604,9 @@ pg_tde_gettup_continue_page(HeapScanDesc scan, ScanDirection dir, int *linesleft /* Caller is responsible for ensuring buffer is locked if needed */ page = BufferGetPage(scan->rs_cbuf); - +#if PG_VERSION_NUM < 170000 TestForOldSnapshot(scan->rs_base.rs_snapshot, scan->rs_base.rs_rd, page); - +#endif if (ScanDirectionIsForward(dir)) { *lineoff = OffsetNumberNext(scan->rs_coffset); @@ -869,8 +871,9 @@ pg_tde_gettup_pagemode(HeapScanDesc scan, /* continue from previously returned page/tuple */ block = scan->rs_cblock; /* current page */ page = BufferGetPage(scan->rs_cbuf); +#if PG_VERSION_NUM < 170000 TestForOldSnapshot(scan->rs_base.rs_snapshot, scan->rs_base.rs_rd, page); - +#endif lineindex = scan->rs_cindex + dir; if (ScanDirectionIsForward(dir)) linesleft = scan->rs_ntuples - lineindex; @@ -889,7 +892,9 @@ pg_tde_gettup_pagemode(HeapScanDesc scan, { pg_tde_getpage((TableScanDesc) scan, block); page = BufferGetPage(scan->rs_cbuf); +#if PG_VERSION_NUM < 170000 TestForOldSnapshot(scan->rs_base.rs_snapshot, scan->rs_base.rs_rd, page); +#endif linesleft = scan->rs_ntuples; lineindex = ScanDirectionIsForward(dir) ? 0 : linesleft - 1; @@ -1377,8 +1382,9 @@ pg_tde_fetch(Relation relation, */ LockBuffer(buffer, BUFFER_LOCK_SHARE); page = BufferGetPage(buffer); +#if PG_VERSION_NUM < 170000 TestForOldSnapshot(snapshot, relation, page); - +#endif /* * We'd better check for out-of-range offnum in case of VACUUM since the * TID was obtained. @@ -1668,8 +1674,9 @@ pg_tde_get_latest_tid(TableScanDesc sscan, buffer = ReadBuffer(relation, ItemPointerGetBlockNumber(&ctid)); LockBuffer(buffer, BUFFER_LOCK_SHARE); page = BufferGetPage(buffer); +#if PG_VERSION_NUM < 170000 TestForOldSnapshot(snapshot, relation, page); - +#endif /* * Check for bogus item number. This is not treated as an error * condition because it can happen while following a t_ctid link. We From b4476405ba197a95f8783ad8227ef6a0dbcb8eaf Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Mon, 15 Apr 2024 19:48:22 +0100 Subject: [PATCH 02/14] Simple SMGR based AES-CBC encryption --- Makefile.in | 1 + meson.build | 2 + src/include/smgr/pg_tde_smgr.h | 4 + src/pg_tde.c | 3 + src/smgr/pg_tde_smgr.c | 155 +++++++++++++++++++++++++++++++++ 5 files changed, 165 insertions(+) create mode 100644 src/include/smgr/pg_tde_smgr.h create mode 100644 src/smgr/pg_tde_smgr.c diff --git a/Makefile.in b/Makefile.in index f5b6d39843c..9894d443012 100644 --- a/Makefile.in +++ b/Makefile.in @@ -44,6 +44,7 @@ src/catalog/tde_keyring.o \ src/catalog/tde_master_key.o \ src/common/pg_tde_shmem.o \ src/common/pg_tde_utils.o \ +src/smgr/pg_tde_smgr.o \ src/pg_tde_defs.o \ src/pg_tde.o diff --git a/meson.build b/meson.build index 366770e00e0..fd15921e023 100644 --- a/meson.build +++ b/meson.build @@ -38,6 +38,8 @@ pg_tde_sources = files( 'src/keyring/keyring_vault.c', 'src/keyring/keyring_api.c', + 'src/smgr/pg_tde_smgr.c', + 'src/catalog/tde_keyring.c', 'src/catalog/tde_master_key.c', 'src/common/pg_tde_shmem.c', diff --git a/src/include/smgr/pg_tde_smgr.h b/src/include/smgr/pg_tde_smgr.h new file mode 100644 index 00000000000..90df06c427f --- /dev/null +++ b/src/include/smgr/pg_tde_smgr.h @@ -0,0 +1,4 @@ + +#pragma once + +extern void RegisterStorageMgr(); \ No newline at end of file diff --git a/src/pg_tde.c b/src/pg_tde.c index 7b076592b02..ed996cffa8d 100644 --- a/src/pg_tde.c +++ b/src/pg_tde.c @@ -31,6 +31,7 @@ #include "keyring/keyring_vault.h" #include "utils/builtins.h" #include "pg_tde_defs.h" +#include "smgr/pg_tde_smgr.h" #define MAX_ON_INSTALLS 5 @@ -98,6 +99,8 @@ _PG_init(void) InstallFileKeyring(); InstallVaultV2Keyring(); RegisterCustomRmgr(RM_TDERMGR_ID, &pg_tde_rmgr); + + RegisterStorageMgr(); } Datum pg_tde_extension_initialize(PG_FUNCTION_ARGS) diff --git a/src/smgr/pg_tde_smgr.c b/src/smgr/pg_tde_smgr.c new file mode 100644 index 00000000000..9e679ec5138 --- /dev/null +++ b/src/smgr/pg_tde_smgr.c @@ -0,0 +1,155 @@ + +#include "smgr/pg_tde_smgr.h" +#include "postgres.h" +#include "storage/smgr.h" +#include "storage/md.h" +#include "catalog/catalog.h" +#include "encryption/enc_aes.h" + +#if PG_VERSION_NUM >= 170000 + +// TODO: implement proper key/IV +static char key[16] = {0,}; +// iv should be based on blocknum, available in the API +static char iv[16] = {0,}; + +void +tde_mdwritev(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + const void **buffers, BlockNumber nblocks, bool skipFsync) +{ + AesInit(); + + char* local_blocks = malloc( BLCKSZ * (nblocks+1) ); + char* local_blocks_aligned = (char*)TYPEALIGN(PG_IO_ALIGN_SIZE, local_blocks); + const void** local_buffers = malloc ( sizeof(void*) * nblocks ); + + // TODO: add check to only encrypt/decrypt tables with specific AM/DB? + + if(IsCatalogRelationOid(reln->smgr_rlocator.locator.spcOid)) + { + // Don't try to encrypt catalog tables: + // Issues with bootstrap and encryption metadata + + mdwritev(reln, forknum, blocknum, buffers, nblocks, skipFsync); + + return; + } + + for(int i = 0; i < nblocks; ++i ) + { + local_buffers[i] = &local_blocks_aligned[i*BLCKSZ]; + int out_len = BLCKSZ; + AesEncrypt(key, iv, ((char**)buffers)[i], BLCKSZ, local_buffers[i], &out_len); + } + + mdwritev(reln, forknum, blocknum, + local_buffers, nblocks, skipFsync); + + free(local_blocks); + free(local_buffers); +} + +void +tde_mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + const void *buffer, bool skipFsync) +{ + AesInit(); + + char* local_blocks = malloc( BLCKSZ * (1+1) ); + char* local_blocks_aligned = (char*)TYPEALIGN(PG_IO_ALIGN_SIZE, local_blocks); + + // TODO: add check to only encrypt/decrypt tables with specific AM/DB? + + if(IsCatalogRelationOid(reln->smgr_rlocator.locator.spcOid)) + { + // Don't try to encrypt catalog tables: + // Issues with bootstrap and encryption metadata + mdextend(reln, forknum, blocknum, buffer, skipFsync); + + return; + } + + int out_len = BLCKSZ; + AesEncrypt(key, iv, ((char*)buffer), BLCKSZ, local_blocks_aligned, &out_len); + + mdextend(reln, forknum, blocknum, local_blocks_aligned, skipFsync); + + + free(local_blocks); +} + +void +tde_mdreadv(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + void **buffers, BlockNumber nblocks) +{ + AesInit(); + + mdreadv(reln, forknum, blocknum, buffers, nblocks); + + // TODO: add check to only encrypt/decrypt tables with specific AM/DB? + + // Don't try to decrypt catalog tables, those are not encrypted + if(IsCatalogRelationOid(reln->smgr_rlocator.locator.spcOid)) + { + return; + } + + for(int i = 0; i < nblocks; ++i) + { + bool allZero = true; + for(int j = 0; j < 32; ++j) + { + if(((char**)buffers)[i][j] != 0) + { + // Postgres creates all zero blocks in an optimized route, which we do not try + // to encrypt. + // Instead we detect if a block is all zero at decryption time, and + // leave it as is. + // This could be a security issue later, but it is a good first prototype + allZero = false; + break; + } + } + if(allZero) continue; + + int out_len = BLCKSZ; + AesDecrypt(key, iv, ((char**)buffers)[i], BLCKSZ, ((char**)buffers)[i], &out_len); + } + + // And now decrypt buffers in place + // We check the first few bytes of the page: if all zero, we assume it is zero and keep it as is +} +static SMgrId tde_smgr_id; +static const struct f_smgr tde_smgr = { + .name = "tde", + .smgr_init = mdinit, + .smgr_shutdown = NULL, + .smgr_open = mdopen, + .smgr_close = mdclose, + .smgr_create = mdcreate, + .smgr_exists = mdexists, + .smgr_unlink = mdunlink, + .smgr_extend = tde_mdextend, + .smgr_zeroextend = mdzeroextend, + .smgr_prefetch = mdprefetch, + .smgr_readv = tde_mdreadv, + .smgr_writev = tde_mdwritev, + .smgr_writeback = mdwriteback, + .smgr_nblocks = mdnblocks, + .smgr_truncate = mdtruncate, + .smgr_immedsync = mdimmedsync, +}; + +void RegisterStorageMgr() +{ + tde_smgr_id = smgr_register(&tde_smgr, 0); + + // TODO: figure out how this part should work in a real extension + storage_manager_id = tde_smgr_id; +} + +#else +void RegisterStorageMgr() +{ +} +#endif \ No newline at end of file From 8973619822ac6f7ea7124d6e968bd81e42fa98d9 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoi Date: Thu, 30 May 2024 15:45:42 +0300 Subject: [PATCH 03/14] XLog encryption prototype (#183) * Encrypts XLog pages while writing them to the segment file. * Add WAL ecryption GUC. * Both streaming and logical replication are working. * TODO: needs key management. --- src/access/pg_tde_xlog.c | 296 +++++++++++++++++++++++++++++++ src/include/access/pg_tde_xlog.h | 26 +++ src/include/pg_tde_defines.h | 1 + src/pg_tde.c | 6 + 4 files changed, 329 insertions(+) diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index b87344e9cb2..ae1a746e3ae 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -12,14 +12,30 @@ #include "postgres.h" +#include "pg_tde_defines.h" #include "access/xlog.h" #include "access/xlog_internal.h" #include "access/xloginsert.h" +#include "storage/bufmgr.h" +#include "storage/shmem.h" +#include "utils/guc.h" +#include "utils/memutils.h" #include "access/pg_tde_tdemap.h" #include "access/pg_tde_xlog.h" #include "catalog/tde_master_key.h" +#include "encryption/enc_tde.h" + +static char *TDEXLogEncryptBuf = NULL; +bool EncryptXLog = false; + +static XLogPageHeaderData EncryptCurrentPageHrd; +static XLogPageHeaderData DecryptCurrentPageHrd; + +static ssize_t TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset); +static void SetXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, char* iv_prefix); +static int XLOGChooseNumBuffers(void); /* * TDE fork XLog */ @@ -103,3 +119,283 @@ pg_tde_rmgr_identify(uint8 info) return NULL; } + +/* + * XLog Storage Manager + * TODO: + * - Should be a config option "on/off"? + * - Currently it encrypts WAL XLog Pages, should we encrypt whole Segments? `initdb` for + * example generates a write of 312 pages - so 312 "gen IV" and "encrypt" runs instead of one. + * Would require though an extra read() during recovery/was_send etc to check `XLogPageHeader` + * if segment is encrypted. + * We could also encrypt Records while adding them to the XLog Buf but it'll be the slowest (?). + */ + +void +xlogInitGUC(void) +{ + DefineCustomBoolVariable("pg_tde.wal_encrypt", /* name */ + "Enable/Disable encryption of WAL.", /* short_desc */ + NULL, /* long_desc */ + &EncryptXLog, /* value address */ + false, /* boot value */ + PGC_POSTMASTER, /* context */ + 0, /* flags */ + NULL, /* check_hook */ + NULL, /* assign_hook */ + NULL /* show_hook */ + ); +} + +static int +XLOGChooseNumBuffers(void) +{ + int xbuffers; + + xbuffers = NBuffers / 32; + if (xbuffers > (wal_segment_size / XLOG_BLCKSZ)) + xbuffers = (wal_segment_size / XLOG_BLCKSZ); + if (xbuffers < 8) + xbuffers = 8; + return xbuffers; +} + +/* + * Defines the size of the XLog encryption buffer + */ +Size +TDEXLogEncryptBuffSize() +{ + int xbuffers; + + xbuffers = (XLOGbuffers == -1) ? XLOGChooseNumBuffers() : XLOGbuffers; + return (Size) XLOG_BLCKSZ * xbuffers; +} + +/* + * Alloc memory for the encryption buffer. + * + * It should fit XLog buffers (XLOG_BLCKSZ * wal_buffers). We can't + * (re)alloc this buf in pg_tde_xlog_seg_write() based on the write size as + * it's called in the CRIT section, hence no allocations are allowed. + * + * Access to this buffer happens during XLogWrite() call which should + * be called with WALWriteLock held, hence no need in extra locks. + */ +void +TDEXLogShmemInit(void) +{ + if (EncryptXLog) + { + bool foundBuf; + + TDEXLogEncryptBuf = (char *) + TYPEALIGN(PG_IO_ALIGN_SIZE, + ShmemInitStruct("TDE XLog Encryption Buffer", + XLOG_TDE_ENC_BUFF_ALIGNED_SIZE, + &foundBuf)); + + elog(DEBUG1, "pg_tde: initialized encryption buffer %lu bytes", XLOG_TDE_ENC_BUFF_ALIGNED_SIZE); + } +} + +void +TDEInitXLogSmgr(void) +{ + SetXLogSmgr(&tde_xlog_smgr); +} + +/* + * TODO: proper key management + * where to store refs to the master and internal keys? + */ +static InternalKey XLogInternalKey = {.key = {0xD,}}; + +ssize_t +pg_tde_xlog_seg_write(int fd, const void *buf, size_t count, off_t offset) +{ + if (EncryptXLog) + return TDEXLogWriteEncryptedPages(fd, buf, count, offset); + else + return pg_pwrite(fd, buf, count, offset); +} + +/* + * Encrypt XLog page(s) from the buf and write to the segment file. + */ +static ssize_t +TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset) +{ + char iv_prefix[16] = {0,}; + size_t data_size = 0; + XLogPageHeader curr_page_hdr = &EncryptCurrentPageHrd; + XLogPageHeader enc_buf_page; + RelKeyData key = {.internal_key = XLogInternalKey}; + off_t enc_off; + size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; + uint32 iv_ctr = 0; + + +#ifdef TDE_XLOG_DEBUG + elog(DEBUG1, "write encrypted WAL, pages amount: %d, size: %lu offset: %ld", count / (Size) XLOG_BLCKSZ, count, offset); +#endif + + /* + * Go through the buf page-by-page and encrypt them. + * We may start or finish writing from/in the middle of the page + * (walsender or `full_page_writes = off`). So preserve a page header + * for the IV init data. + * + * TODO: check if walsender restarts form the beggining of the page + * in case of the crash. + */ + for (enc_off = 0; enc_off < count;) + { + data_size = Min(page_size, count); + + if (page_size == XLOG_BLCKSZ) + { + memcpy((char *) curr_page_hdr, (char *) buf + enc_off, SizeOfXLogShortPHD); + + /* + * Need to use a separate buf for the encryption so the page remains non-crypted + * in the XLog buf (XLogInsert has to have access to records' lsn). + */ + enc_buf_page = (XLogPageHeader) (TDEXLogEncryptBuf + enc_off); + memcpy((char *) enc_buf_page, (char *) buf + enc_off, (Size) XLogPageHeaderSize(curr_page_hdr)); + enc_buf_page->xlp_info |= XLP_ENCRYPTED; + + enc_off += XLogPageHeaderSize(curr_page_hdr); + data_size -= XLogPageHeaderSize(curr_page_hdr); + /* it's a beginning of the page */ + iv_ctr = 0; + } + else + { + /* we're in the middle of the page */ + iv_ctr = (offset % XLOG_BLCKSZ) - XLogPageHeaderSize(curr_page_hdr); + } + + if (data_size + enc_off > count) + { + data_size = count - enc_off; + } + + /* + * The page is zeroed (no data), no sense to enctypt. + * This may happen when base_backup or other requests XLOG SWITCH and + * some pages in XLog buffer still not used. + */ + if (curr_page_hdr->xlp_magic == 0) + { + /* ensure all the page is {0} */ + Assert((*((char *) buf + enc_off) == 0) && + memcmp((char *) buf + enc_off, (char *) buf + enc_off + 1, data_size - 1) == 0); + + memcpy((char *) enc_buf_page, (char *) buf + enc_off, data_size); + } + else + { + SetXLogPageIVPrefix(curr_page_hdr->xlp_tli, curr_page_hdr->xlp_pageaddr, iv_prefix); + PG_TDE_ENCRYPT_DATA(iv_prefix, iv_ctr, (char *) buf + enc_off, data_size, + TDEXLogEncryptBuf + enc_off, &key); + } + + page_size = XLOG_BLCKSZ; + enc_off += data_size; + } + + return pg_pwrite(fd, TDEXLogEncryptBuf, count, offset); +} + +/* + * Read the XLog pages from the segment file and dectypt if need. + */ +ssize_t +pg_tde_xlog_seg_read(int fd, void *buf, size_t count, off_t offset) +{ + ssize_t readsz; + char iv_prefix[16] = {0,}; + size_t data_size = 0; + XLogPageHeader curr_page_hdr = &DecryptCurrentPageHrd; + RelKeyData key = {.internal_key = XLogInternalKey}; + size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; + off_t dec_off; + uint32 iv_ctr = 0; + +#ifdef TDE_XLOG_DEBUG + elog(DEBUG1, "read from a WAL segment, pages amount: %d, size: %lu offset: %ld", count / (Size) XLOG_BLCKSZ, count, offset); +#endif + + readsz = pg_pread(fd, buf, count, offset); + + /* + * Read the buf page by page and decypt ecnrypted pages. + * We may start or fihish reading from/in the middle of the page (walreceiver) + * in such a case we should preserve the last read page header for + * the IV data and the encryption state. + * + * TODO: check if walsender/receiver restarts form the beggining of the page + * in case of the crash. + */ + for (dec_off = 0; dec_off < readsz;) + { + data_size = Min(page_size, readsz); + + if (page_size == XLOG_BLCKSZ) + { + memcpy((char *) curr_page_hdr, (char *) buf + dec_off, SizeOfXLogShortPHD); + + /* set the flag to "not encrypted" for the walreceiver */ + ((XLogPageHeader) ((char *) buf + dec_off))->xlp_info &= ~XLP_ENCRYPTED; + + Assert(curr_page_hdr->xlp_magic == XLOG_PAGE_MAGIC || curr_page_hdr->xlp_magic == 0); + dec_off += XLogPageHeaderSize(curr_page_hdr); + data_size -= XLogPageHeaderSize(curr_page_hdr); + /* it's a beginning of the page */ + iv_ctr = 0; + } + else + { + /* we're in the middle of the page */ + iv_ctr = (offset % XLOG_BLCKSZ) - XLogPageHeaderSize(curr_page_hdr); + } + + if ((data_size + dec_off) > readsz) + { + data_size = readsz - dec_off; + } + + if (curr_page_hdr->xlp_info & XLP_ENCRYPTED) + { + SetXLogPageIVPrefix(curr_page_hdr->xlp_tli, curr_page_hdr->xlp_pageaddr, iv_prefix); + PG_TDE_DECRYPT_DATA( + iv_prefix, iv_ctr, + (char *) buf + dec_off, data_size, (char *) buf + dec_off, &key); + } + + page_size = XLOG_BLCKSZ; + dec_off += data_size; + } + + return readsz; +} + +/* IV: TLI(uint32) + XLogRecPtr(uint64)*/ +static void +SetXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, char* iv_prefix) +{ + iv_prefix[0] = (tli >> 24); + iv_prefix[1] = ((tli >> 16) & 0xFF); + iv_prefix[2] = ((tli >> 8) & 0xFF); + iv_prefix[3] = (tli & 0xFF); + + iv_prefix[4] = (lsn >> 56); + iv_prefix[5] = ((lsn >> 48) & 0xFF); + iv_prefix[6] = ((lsn >> 40) & 0xFF); + iv_prefix[7] = ((lsn >> 32) & 0xFF); + iv_prefix[8] = ((lsn >> 24) & 0xFF); + iv_prefix[9] = ((lsn >> 16) & 0xFF); + iv_prefix[10] = ((lsn >> 8) & 0xFF); + iv_prefix[11] = (lsn & 0xFF); +} \ No newline at end of file diff --git a/src/include/access/pg_tde_xlog.h b/src/include/access/pg_tde_xlog.h index bc32c979457..8317fd444a6 100644 --- a/src/include/access/pg_tde_xlog.h +++ b/src/include/access/pg_tde_xlog.h @@ -9,7 +9,10 @@ #ifndef PG_TDE_XLOG_H #define PG_TDE_XLOG_H +#include "postgres.h" +#include "access/xlog.h" #include "access/xlog_internal.h" +#include "access/xlog_smgr.h" /* TDE XLOG resource manager */ #define XLOG_TDE_ADD_RELATION_KEY 0x00 @@ -32,4 +35,27 @@ static const RmgrData pg_tde_rmgr = { .rm_identify = pg_tde_rmgr_identify }; +/* XLog encryption staff */ + +/* GUC */ +extern bool EncryptXLog; + +extern Size TDEXLogEncryptBuffSize(); + +#define XLOG_TDE_ENC_BUFF_ALIGNED_SIZE add_size(TDEXLogEncryptBuffSize(), PG_IO_ALIGN_SIZE) + +extern void TDEXLogShmemInit(void); + +extern ssize_t pg_tde_xlog_seg_read(int fd, void *buf, size_t count, off_t offset); +extern ssize_t pg_tde_xlog_seg_write(int fd, const void *buf, size_t count, off_t offset); + +static const XLogSmgr tde_xlog_smgr = { + .seg_read = pg_tde_xlog_seg_read, + .seg_write = pg_tde_xlog_seg_write, +}; + +extern void TDEInitXLogSmgr(void); + +extern void xlogInitGUC(void); + #endif /* PG_TDE_XLOG_H */ diff --git a/src/include/pg_tde_defines.h b/src/include/pg_tde_defines.h index aaa497226c2..0c9847bb7df 100644 --- a/src/include/pg_tde_defines.h +++ b/src/include/pg_tde_defines.h @@ -22,6 +22,7 @@ //#define ENCRYPTION_DEBUG 1 //#define KEYRING_DEBUG 1 //#define TDE_FORK_DEBUG 1 +// #define TDE_XLOG_DEBUG 1 #define pg_tde_fill_tuple heap_fill_tuple #define pg_tde_form_tuple heap_form_tuple diff --git a/src/pg_tde.c b/src/pg_tde.c index ed996cffa8d..b676b1255c0 100644 --- a/src/pg_tde.c +++ b/src/pg_tde.c @@ -60,6 +60,9 @@ tde_shmem_request(void) { Size sz = TdeRequiredSharedMemorySize(); int required_locks = TdeRequiredLocksCount(); + + sz = add_size(sz, XLOG_TDE_ENC_BUFF_ALIGNED_SIZE); + if (prev_shmem_request_hook) prev_shmem_request_hook(); RequestAddinShmemSpace(sz); @@ -75,6 +78,8 @@ tde_shmem_startup(void) TdeShmemInit(); AesInit(); + TDEXLogShmemInit(); + TDEInitXLogSmgr(); } void @@ -87,6 +92,7 @@ _PG_init(void) keyringRegisterVariables(); InitializeMasterKeyInfo(); + xlogInitGUC(); prev_shmem_request_hook = shmem_request_hook; shmem_request_hook = tde_shmem_request; From 48d211926d2af5078a160bf86bf2dc2a170772ad Mon Sep 17 00:00:00 2001 From: Muhammad Usama Date: Tue, 11 Jun 2024 23:26:44 +0500 Subject: [PATCH 04/14] Implement event triggers to identify index creation on encrypted tables. (#196) This commit implements ddl-start and ddl-end event triggers to identify index creation operations on encrypted tables. Upon creating an index on an encrypted table, the trigger function updates the global state, which can be accessed by the storage manager (mgr) to decide if smgr_create needs to do encryption or not. The start-ddl function analyzes the CREATE TABLE and CREATE INDEX statements and identifies if the table uses the pg_tde access method. When the table is created or the one on which the index is being created utilizes the pg_tde access method, the start-ddl trigger function populates relevant information about the encrypted table into a global structure. This structure can be accessed using the GetCurrentTdeCreateEvent() function. After the execution of the current DDL command finishes, the end-ddl function clears out this structure. --- Makefile.in | 1 + meson.build | 1 + pg_tde--1.0.sql | 18 ++++ src/include/pg_tde_event_capture.h | 33 +++++++ src/pg_tde_event_capture.c | 147 +++++++++++++++++++++++++++++ 5 files changed, 200 insertions(+) create mode 100644 src/include/pg_tde_event_capture.h create mode 100644 src/pg_tde_event_capture.c diff --git a/Makefile.in b/Makefile.in index 9894d443012..8f530f2dc7f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -46,6 +46,7 @@ src/common/pg_tde_shmem.o \ src/common/pg_tde_utils.o \ src/smgr/pg_tde_smgr.o \ src/pg_tde_defs.o \ +src/pg_tde_event_capture.o \ src/pg_tde.o override PG_CPPFLAGS += @tde_CPPFLAGS@ diff --git a/meson.build b/meson.build index fd15921e023..b9cb1e87184 100644 --- a/meson.build +++ b/meson.build @@ -46,6 +46,7 @@ pg_tde_sources = files( 'src/common/pg_tde_utils.c', 'src/pg_tde_defs.c', 'src/pg_tde.c', + 'src/pg_tde_event_capture.c', ) incdir = include_directories('src/include', '.') diff --git a/pg_tde--1.0.sql b/pg_tde--1.0.sql index 3d32ec5e7cb..4474e7496cb 100644 --- a/pg_tde--1.0.sql +++ b/pg_tde--1.0.sql @@ -131,3 +131,21 @@ COMMENT ON ACCESS METHOD pg_tde IS 'pg_tde table access method'; -- Per database extension initialization SELECT pg_tde_extension_initialize(); + +CREATE OR REPLACE FUNCTION pg_tde_ddl_command_start_capture() +RETURNS event_trigger +AS 'MODULE_PATHNAME' +LANGUAGE C; + +CREATE OR REPLACE FUNCTION pg_tde_ddl_command_end_capture() +RETURNS event_trigger +AS 'MODULE_PATHNAME' +LANGUAGE C; + +CREATE EVENT TRIGGER pg_tde_trigger_create_index +ON ddl_command_start +EXECUTE FUNCTION pg_tde_ddl_command_start_capture(); + +CREATE EVENT TRIGGER pg_tde_trigger_create_index_2 +ON ddl_command_end +EXECUTE FUNCTION pg_tde_ddl_command_end_capture(); diff --git a/src/include/pg_tde_event_capture.h b/src/include/pg_tde_event_capture.h new file mode 100644 index 00000000000..d93226ffec1 --- /dev/null +++ b/src/include/pg_tde_event_capture.h @@ -0,0 +1,33 @@ +/*------------------------------------------------------------------------- + * + * pg_tde_event_capture.h + * + *------------------------------------------------------------------------- + */ +#ifndef PG_TDE_EVENT_CAPTURE_H +#define PG_TDE_EVENT_CAPTURE_H + +#include "postgres.h" +#include "nodes/parsenodes.h" + +typedef enum TdeCreateEventType +{ + TDE_UNKNOWN_CREATE_EVENT, + TDE_TABLE_CREATE_EVENT, + TDE_INDEX_CREATE_EVENT +} TdeCreateEventType; + +typedef struct TdeCreateEvent +{ + TdeCreateEventType eventType; /* DDL statement type */ + bool encryptMode; /* true when the table uses encryption */ + Oid baseTableOid; /* Oid of table on which index is being + * created on. For create table statement this + * contains InvalidOid */ + RangeVar *relation; /* Reference to the parsed relation from + * create statement */ +} TdeCreateEvent; + +extern TdeCreateEvent * GetCurrentTdeCreateEvent(void); + +#endif diff --git a/src/pg_tde_event_capture.c b/src/pg_tde_event_capture.c new file mode 100644 index 00000000000..1797ec5773c --- /dev/null +++ b/src/pg_tde_event_capture.c @@ -0,0 +1,147 @@ +/*------------------------------------------------------------------------- + * + * pg_tde_event_capture.c + * event trigger logic to identify if we are creating the encrypted table or not. + * + * IDENTIFICATION + * contrib/pg_tde/src/pg_tde_event_trigger.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "funcapi.h" +#include "fmgr.h" +#include "utils/rel.h" +#include "utils/builtins.h" +#include "catalog/pg_class.h" +#include "access/table.h" +#include "catalog/pg_event_trigger.h" +#include "catalog/namespace.h" +#include "commands/event_trigger.h" +#include "common/pg_tde_utils.h" +#include "pg_tde_event_capture.h" + +/* Global variable that gets set at ddl start and cleard out at ddl end*/ +TdeCreateEvent tdeCurrentCreateEvent = {.relation = NULL}; + + +static void reset_current_tde_create_event(void); + +PG_FUNCTION_INFO_V1(pg_tde_ddl_command_start_capture); +PG_FUNCTION_INFO_V1(pg_tde_ddl_command_end_capture); + +TdeCreateEvent * +GetCurrentTdeCreateEvent(void) +{ + return &tdeCurrentCreateEvent; +} + +/* + * pg_tde_ddl_command_start_capture is an event trigger function triggered + * at the start of any DDL command execution. + * + * The function specifically focuses on CREATE INDEX and CREATE TABLE statements, + * aiming to determine if the create table or the table on which an index is being created + * utilizes the pg_tde access method for encryption. + * Once it confirms the table's encryption requirement or usage, + * it updates the table information in the tdeCurrentCreateEvent global variable. + * This information can be accessed by SMGR or any other component + * during the execution of this DDL statement. + */ +Datum +pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) +{ + EventTriggerData *trigdata; + Node *parsetree; + + /* Ensure this function is being called as an event trigger */ + if (!CALLED_AS_EVENT_TRIGGER(fcinfo)) /* internal error */ + ereport(ERROR, + (errmsg("Function can only be fired by event trigger manager"))); + + trigdata = (EventTriggerData *) fcinfo->context; + parsetree = trigdata->parsetree; + + elog(DEBUG2, "EVENT TRIGGER (%s) %s", trigdata->event, nodeToString(parsetree)); + reset_current_tde_create_event(); + + if (IsA(parsetree, IndexStmt)) + { + IndexStmt *stmt = (IndexStmt *) parsetree; + Oid relationId = RangeVarGetRelid(stmt->relation, NoLock, true); + + tdeCurrentCreateEvent.eventType = TDE_INDEX_CREATE_EVENT; + tdeCurrentCreateEvent.baseTableOid = relationId; + tdeCurrentCreateEvent.relation = stmt->relation; + + if (relationId != InvalidOid) + { + LOCKMODE lockmode = AccessShareLock; /* TODO. Verify lock mode? */ + Relation rel = table_open(relationId, lockmode); + + if (rel->rd_rel->relam == get_tde_table_am_oid()) + { + /* We are creating the index on encrypted table */ + ereport(NOTICE, + (errmsg("Creating index on **ENCRYPTED** relation:%s with Oid:%d", stmt->relation->relname, relationId))); + /* set the global state */ + tdeCurrentCreateEvent.encryptMode = true; + } + else + ereport(DEBUG1, + (errmsg("Creating index on relation:%s with Oid:%d", stmt->relation->relname, relationId))); + table_close(rel, lockmode); + } + else + ereport(DEBUG1, (errmsg("Failed to get relation Oid for relation:%s", stmt->relation->relname))); + + } + else if (IsA(parsetree, CreateStmt)) + { + CreateStmt *stmt = (CreateStmt *) parsetree; + + tdeCurrentCreateEvent.eventType = TDE_TABLE_CREATE_EVENT; + tdeCurrentCreateEvent.relation = stmt->relation; + + elog(DEBUG1, "CREATING TABLE %s Using Access Method %s", stmt->relation->relname, stmt->accessMethod); + if (stmt->accessMethod && !strcmp(stmt->accessMethod, "pg_tde")) + { + tdeCurrentCreateEvent.encryptMode = true; + } + } + PG_RETURN_NULL(); +} + +/* + * trigger function called at the end of DDL statement execution. + * It just clears the tdeCurrentCreateEvent global variable. + */ +Datum +pg_tde_ddl_command_end_capture(PG_FUNCTION_ARGS) +{ + /* Ensure this function is being called as an event trigger */ + if (!CALLED_AS_EVENT_TRIGGER(fcinfo)) /* internal error */ + ereport(ERROR, + (errmsg("Function can only be fired by event trigger manager"))); + + elog(DEBUG1, "Type:%s EncryptMode:%s, Oid:%d, Relation:%s ", + (tdeCurrentCreateEvent.eventType == TDE_INDEX_CREATE_EVENT) ? "CREATE INDEX" : + (tdeCurrentCreateEvent.eventType == TDE_TABLE_CREATE_EVENT) ? "CREATE TABLE" : "UNKNOWN", + tdeCurrentCreateEvent.encryptMode ? "true" : "false", + tdeCurrentCreateEvent.baseTableOid, + tdeCurrentCreateEvent.relation ? tdeCurrentCreateEvent.relation->relname : "UNKNOWN"); + + /* All we need to do is to clear the event state */ + reset_current_tde_create_event(); + PG_RETURN_NULL(); +} + +static void +reset_current_tde_create_event(void) +{ + tdeCurrentCreateEvent.encryptMode = false; + tdeCurrentCreateEvent.eventType = TDE_UNKNOWN_CREATE_EVENT; + tdeCurrentCreateEvent.baseTableOid = InvalidOid; + tdeCurrentCreateEvent.relation = NULL; +} From 893512584461027c7922123dbfb0f94bdeaa9a92 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Tue, 11 Jun 2024 20:02:51 +0100 Subject: [PATCH 05/14] Minimal working prototype for SMGR + Event trigger + keyring encryption (#199) * Introduces `pg_tde2` access method * New access method uses the event trigger changes from #196 * Keys are now loaded from the keyring * This required changes to the map file / master key infrastructure * This commit only modifies/fixes those as little as required for simplicity. More refactoring/changes coming in separate commits / PRs * Removes reliance from MyDatabaseId, as things now have to work with multiple databases in the checkpointer * Removes some error reports, where functions should work even without a configured keyring * Fixes some bugs in the map file functions * Map file functions now work with multiple databases in a single process, but this is a hackish solution, global state needs a proper refactoring * Contains anti-recursion hack in the new SMGR code, which is needed until we store the metadata in the catalog --- pg_tde--1.0.sql | 9 +++ src/access/pg_tde_tdemap.c | 40 ++++++---- src/access/pg_tdeam_handler.c | 14 ++-- src/catalog/tde_master_key.c | 14 +--- src/common/pg_tde_utils.c | 8 +- src/include/access/pg_tde_tdemap.h | 3 +- src/include/catalog/tde_master_key.h | 2 +- src/include/common/pg_tde_utils.h | 3 +- src/include/pg_tde_defines.h | 2 - src/pg_tde_event_capture.c | 2 +- src/smgr/pg_tde_smgr.c | 106 +++++++++++++++++++++------ t/results/001_basic.out | 13 ---- 12 files changed, 142 insertions(+), 74 deletions(-) delete mode 100644 t/results/001_basic.out diff --git a/pg_tde--1.0.sql b/pg_tde--1.0.sql index 4474e7496cb..9d43e98a939 100644 --- a/pg_tde--1.0.sql +++ b/pg_tde--1.0.sql @@ -87,6 +87,12 @@ RETURNS table_am_handler AS 'MODULE_PATHNAME' LANGUAGE C; +-- Table access method +CREATE FUNCTION pg_tde2am_handler(internal) +RETURNS table_am_handler +AS 'MODULE_PATHNAME' +LANGUAGE C; + CREATE FUNCTION pgtde_is_encrypted(table_name VARCHAR) RETURNS boolean AS $$ @@ -129,6 +135,9 @@ CREATE FUNCTION pg_tde_version() RETURNS TEXT AS 'MODULE_PATHNAME' LANGUAGE C; CREATE ACCESS METHOD pg_tde TYPE TABLE HANDLER pg_tdeam_handler; COMMENT ON ACCESS METHOD pg_tde IS 'pg_tde table access method'; +CREATE ACCESS METHOD pg_tde2 TYPE TABLE HANDLER pg_tde2am_handler; +COMMENT ON ACCESS METHOD pg_tde2 IS 'pg_tde2 table access method'; + -- Per database extension initialization SELECT pg_tde_extension_initialize(); diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 0f3b1307b0a..ced8f6a26d1 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -108,8 +108,8 @@ static void finalize_key_rotation(char *m_path_old, char *k_path_old, char *m_pa /* * Generate an encrypted key for the relation and store it in the keymap file. */ -void -pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, Relation rel) +RelKeyData* +pg_tde_create_key_map_entry(const RelFileLocator *newrlocator) { InternalKey int_key; RelKeyData *rel_key_data; @@ -117,11 +117,14 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, Relation rel) TDEMasterKey *master_key; XLogRelKey xlrec; - master_key = GetMasterKey(); + pg_tde_set_db_file_paths(newrlocator->dbOid); + master_key = GetMasterKey(newrlocator->dbOid); if (master_key == NULL) { ereport(ERROR, (errmsg("failed to retrieve master key"))); + + return NULL; } memset(&int_key, 0, sizeof(InternalKey)); @@ -131,7 +134,9 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, Relation rel) ereport(FATAL, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not generate internal key for relation \"%s\": %s", - RelationGetRelationName(rel), ERR_error_string(ERR_get_error(), NULL)))); + "TODO", ERR_error_string(ERR_get_error(), NULL)))); + + return NULL; } /* Encrypt the key */ @@ -152,6 +157,8 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, Relation rel) * Add the encyrpted key to the key map data file structure. */ pg_tde_write_key_map_entry(newrlocator, enc_rel_key_data, &master_key->keyInfo); + + return rel_key_data; } /* Head of the key cache (linked list) */ @@ -179,7 +186,10 @@ GetRelationKey(RelFileLocator rel) key = pg_tde_get_key_from_file(&rel); - put_key_into_map(rel.relNumber, key); + if (key != NULL) + { + put_key_into_map(rel.relNumber, key); + } return key; } @@ -269,8 +279,9 @@ static void pg_tde_set_db_file_paths(Oid dbOid) { /* Return if the values are already set */ - if (*db_path && *db_map_path && *db_keydata_path) - return; + // TODO: remove this entire global state thing + //if (*db_path && *db_map_path && *db_keydata_path) + // return; /* Fill in the values */ snprintf(db_path, MAXPGPATH, "%s", GetDatabasePath(dbOid, DEFAULTTABLESPACE_OID)); @@ -953,7 +964,8 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator) LWLockAcquire(lock_files, LW_SHARED); /* Get/generate a master, create the key for relation and get the encrypted key with bytes to write */ - master_key = GetMasterKey(); + pg_tde_set_db_file_paths(rlocator->dbOid); + master_key = GetMasterKey(rlocator->dbOid); if (master_key == NULL) { LWLockRelease(lock_files); @@ -961,13 +973,15 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator) (errmsg("failed to retrieve master key"))); } - /* Get the file paths */ - pg_tde_set_db_file_paths(rlocator->dbOid); - /* Read the map entry and get the index of the relation key */ key_index = pg_tde_process_map_entry(rlocator, db_map_path, &offset, false); - /* Add the encrypted key to the data file. */ + if (key_index == -1) + { + LWLockRelease(lock_files); + return NULL; + } + enc_rel_key_data = pg_tde_read_keydata(db_keydata_path, key_index, master_key); LWLockRelease(lock_files); @@ -1198,4 +1212,4 @@ pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_ LWLockRelease(lock_files); return true; -} \ No newline at end of file +} diff --git a/src/access/pg_tdeam_handler.c b/src/access/pg_tdeam_handler.c index a944c4e21e3..0c697b06621 100644 --- a/src/access/pg_tdeam_handler.c +++ b/src/access/pg_tdeam_handler.c @@ -53,6 +53,7 @@ #include "utils/rel.h" PG_FUNCTION_INFO_V1(pg_tdeam_handler); +PG_FUNCTION_INFO_V1(pg_tde2am_handler); static void reform_and_rewrite_tuple(HeapTuple tuple, @@ -643,7 +644,7 @@ pg_tdeam_relation_set_new_filelocator(Relation rel, ereport(DEBUG1, (errmsg("creating key file for relation %s", RelationGetRelationName(rel)))); - pg_tde_create_key_map_entry(newrlocator, rel); + pg_tde_create_key_map_entry(newrlocator); } } @@ -2622,17 +2623,16 @@ static const TableAmRoutine pg_tdeam_methods = { .scan_sample_next_tuple = pg_tdeam_scan_sample_next_tuple }; - -const TableAmRoutine * -GetPGTdeamTableAmRoutine(void) +Datum +pg_tdeam_handler(PG_FUNCTION_ARGS) { - return &pg_tdeam_methods; + PG_RETURN_POINTER(&pg_tdeam_methods); } Datum -pg_tdeam_handler(PG_FUNCTION_ARGS) +pg_tde2am_handler(PG_FUNCTION_ARGS) { - PG_RETURN_POINTER(&pg_tdeam_methods); + PG_RETURN_POINTER(GetHeapamTableAmRoutine()); } bool diff --git a/src/catalog/tde_master_key.c b/src/catalog/tde_master_key.c index 2d2cd9cda35..610f9a6c5e5 100644 --- a/src/catalog/tde_master_key.c +++ b/src/catalog/tde_master_key.c @@ -214,14 +214,13 @@ save_master_key_info(TDEMasterKeyInfo *master_key_info) * throws an error. */ TDEMasterKey * -GetMasterKey(void) +GetMasterKey(Oid dbOid) { TDEMasterKey *masterKey = NULL; TDEMasterKeyInfo *masterKeyInfo = NULL; GenericKeyring *keyring = NULL; const keyInfo *keyInfo = NULL; KeyringReturnCodes keyring_ret; - Oid dbOid = MyDatabaseId; LWLock *lock_files = tde_lwlock_mk_files(); LWLock *lock_cache = tde_lwlock_mk_cache(); @@ -255,9 +254,6 @@ GetMasterKey(void) LWLockRelease(lock_cache); LWLockRelease(lock_files); - ereport(ERROR, - (errmsg("Master key does not exists for the database"), - errhint("Use set_master_key interface to set the master key"))); return NULL; } @@ -268,8 +264,6 @@ GetMasterKey(void) LWLockRelease(lock_cache); LWLockRelease(lock_files); - ereport(ERROR, - (errmsg("Key provider with ID:\"%d\" does not exists", masterKeyInfo->keyringId))); return NULL; } @@ -279,8 +273,6 @@ GetMasterKey(void) LWLockRelease(lock_cache); LWLockRelease(lock_files); - ereport(ERROR, - (errmsg("failed to retrieve master key \"%s\" from keyring.", masterKeyInfo->keyId.versioned_name))); return NULL; } @@ -404,7 +396,7 @@ SetMasterKey(const char *key_name, const char *provider_name, bool ensure_new_ke bool RotateMasterKey(const char *new_key_name, const char *new_provider_name, bool ensure_new_key) { - TDEMasterKey *master_key = GetMasterKey(); + TDEMasterKey *master_key = GetMasterKey(MyDatabaseId); TDEMasterKey new_master_key; const keyInfo *keyInfo = NULL; GenericKeyring *keyring; @@ -737,7 +729,7 @@ Datum pg_tde_master_key_info(PG_FUNCTION_ARGS) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context that cannot accept type record"))); - master_key = GetMasterKey(); + master_key = GetMasterKey(MyDatabaseId); if (master_key == NULL) PG_RETURN_NULL(); diff --git a/src/common/pg_tde_utils.c b/src/common/pg_tde_utils.c index 5f5c9e938f4..af359693ac4 100644 --- a/src/common/pg_tde_utils.c +++ b/src/common/pg_tde_utils.c @@ -33,6 +33,12 @@ get_tde_table_am_oid(void) return get_table_am_oid("pg_tde", false); } +Oid +get_tde2_table_am_oid(void) +{ + return get_table_am_oid("pg_tde2", false); +} + /* * Returns the list of OIDs for all TDE tables in a database */ @@ -208,4 +214,4 @@ extract_json_option_value(Datum top_json, const char* field_name) elog(ERROR, "Unknown type for object %s: %s", field_name, type_cstr); return NULL; } -} \ No newline at end of file +} diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index c6ff9083f1c..a0b4a899e2a 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -46,12 +46,11 @@ typedef struct XLogRelKey RelKeyData relKey; } XLogRelKey; -extern void pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, Relation rel); +extern RelKeyData* pg_tde_create_key_map_entry(const RelFileLocator *newrlocator); extern void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_key_data, TDEMasterKeyInfo *master_key_info); extern void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator); extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset); -extern RelKeyData *pg_tde_get_key_from_fork(const RelFileLocator *rlocator); extern RelKeyData *GetRelationKey(RelFileLocator rel); extern void pg_tde_cleanup_path_vars(void); diff --git a/src/include/catalog/tde_master_key.h b/src/include/catalog/tde_master_key.h index b12e2b1f5f8..77bad07f11c 100644 --- a/src/include/catalog/tde_master_key.h +++ b/src/include/catalog/tde_master_key.h @@ -68,7 +68,7 @@ extern LWLock *tde_lwlock_mk_cache(void); extern bool save_master_key_info(TDEMasterKeyInfo *masterKeyInfo); extern Oid GetMasterKeyProviderId(void); -extern TDEMasterKey* GetMasterKey(void); +extern TDEMasterKey* GetMasterKey(Oid); extern bool SetMasterKey(const char *key_name, const char *provider_name, bool ensure_new_key); extern bool RotateMasterKey(const char *new_key_name, const char *new_provider_name, bool ensure_new_key); extern bool xl_tde_perform_rotate_key(XLogMasterKeyRotate *xlrec); diff --git a/src/include/common/pg_tde_utils.h b/src/include/common/pg_tde_utils.h index c33c14b6235..9e2d856e993 100644 --- a/src/include/common/pg_tde_utils.h +++ b/src/include/common/pg_tde_utils.h @@ -12,10 +12,11 @@ #include "nodes/pg_list.h" extern Oid get_tde_table_am_oid(void); +extern Oid get_tde2_table_am_oid(void); extern List *get_all_tde_tables(void); extern int get_tde_tables_count(void); extern const char *extract_json_cstr(Datum json, const char* field_name); const char *extract_json_option_value(Datum top_json, const char* field_name); -#endif /*PG_TDE_UTILS_H*/ \ No newline at end of file +#endif /*PG_TDE_UTILS_H*/ diff --git a/src/include/pg_tde_defines.h b/src/include/pg_tde_defines.h index 0c9847bb7df..74f47f9db16 100644 --- a/src/include/pg_tde_defines.h +++ b/src/include/pg_tde_defines.h @@ -38,8 +38,6 @@ #define pgstat_count_pg_tde_insert pgstat_count_heap_insert #define pg_tde_getattr heap_getattr -#define GetPGTdeamTableAmRoutine GetHeapamTableAmRoutine - #define TDE_PageAddItem(rel, oid, blkno, page, item, size, offsetNumber, overwrite, is_heap) \ PGTdePageAddItemExtended(rel, oid, blkno, page, item, size, offsetNumber, \ ((overwrite) ? PAI_OVERWRITE : 0) | \ diff --git a/src/pg_tde_event_capture.c b/src/pg_tde_event_capture.c index 1797ec5773c..3645b45e1ea 100644 --- a/src/pg_tde_event_capture.c +++ b/src/pg_tde_event_capture.c @@ -105,7 +105,7 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) tdeCurrentCreateEvent.relation = stmt->relation; elog(DEBUG1, "CREATING TABLE %s Using Access Method %s", stmt->relation->relname, stmt->accessMethod); - if (stmt->accessMethod && !strcmp(stmt->accessMethod, "pg_tde")) + if (stmt->accessMethod && !strcmp(stmt->accessMethod, "pg_tde2")) { tdeCurrentCreateEvent.encryptMode = true; } diff --git a/src/smgr/pg_tde_smgr.c b/src/smgr/pg_tde_smgr.c index 9e679ec5138..ceb83cfa186 100644 --- a/src/smgr/pg_tde_smgr.c +++ b/src/smgr/pg_tde_smgr.c @@ -5,14 +5,72 @@ #include "storage/md.h" #include "catalog/catalog.h" #include "encryption/enc_aes.h" +#include "access/pg_tde_tdemap.h" +#include "pg_tde_event_capture.h" #if PG_VERSION_NUM >= 170000 -// TODO: implement proper key/IV -static char key[16] = {0,}; -// iv should be based on blocknum, available in the API +// TODO: implement proper IV +// iv should be based on blocknum + relfile, available in the API static char iv[16] = {0,}; +static RelKeyData* +tde_smgr_get_key(SMgrRelation reln) +{ + // TODO: This recursion counter is a dirty hack until the metadata is in the catalog + // As otherwise we would call GetMasterKey recursively and deadlock + static int recursion = 0; + + ereport(NOTICE, + (errmsg("Trying to decide if table is encrypted: %u", reln->smgr_rlocator.locator.relNumber))); + + + if(IsCatalogRelationOid(reln->smgr_rlocator.locator.relNumber)) + { + // do not try to encrypt/decrypt catalog tables + return NULL; + } + + if(recursion != 0) + { + return NULL; + } + + recursion++; + + + if(GetMasterKey(reln->smgr_rlocator.locator.relNumber)==NULL) + { + recursion--; + return NULL; + } + + TdeCreateEvent* event = GetCurrentTdeCreateEvent(); + + // if this is a CREATE TABLE, we have to generate the key + if(event->encryptMode == true && event->eventType == TDE_TABLE_CREATE_EVENT) + { + recursion--; + return pg_tde_create_key_map_entry(&reln->smgr_rlocator.locator); + } + + // if this is a CREATE INDEX, we have to load the key based on the table + if(event->encryptMode == true && event->eventType == TDE_INDEX_CREATE_EVENT) + { + // For now keep it simple and create separate key for indexes + // Later we might modify the map infrastructure to support the same keys + recursion--; + return pg_tde_create_key_map_entry(&reln->smgr_rlocator.locator); + } + + // otherwise, see if we have a key for the relation, and return if yes + RelKeyData* rkd = GetRelationKey(reln->smgr_rlocator.locator); + + recursion--; + + return rkd; +} + void tde_mdwritev(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, const void **buffers, BlockNumber nblocks, bool skipFsync) @@ -23,13 +81,10 @@ tde_mdwritev(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char* local_blocks_aligned = (char*)TYPEALIGN(PG_IO_ALIGN_SIZE, local_blocks); const void** local_buffers = malloc ( sizeof(void*) * nblocks ); - // TODO: add check to only encrypt/decrypt tables with specific AM/DB? + RelKeyData* rkd = tde_smgr_get_key(reln); - if(IsCatalogRelationOid(reln->smgr_rlocator.locator.spcOid)) + if(rkd == NULL) { - // Don't try to encrypt catalog tables: - // Issues with bootstrap and encryption metadata - mdwritev(reln, forknum, blocknum, buffers, nblocks, skipFsync); return; @@ -39,7 +94,7 @@ tde_mdwritev(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, { local_buffers[i] = &local_blocks_aligned[i*BLCKSZ]; int out_len = BLCKSZ; - AesEncrypt(key, iv, ((char**)buffers)[i], BLCKSZ, local_buffers[i], &out_len); + AesEncrypt(rkd->internal_key.key, iv, ((char**)buffers)[i], BLCKSZ, local_buffers[i], &out_len); } mdwritev(reln, forknum, blocknum, @@ -58,19 +113,17 @@ tde_mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char* local_blocks = malloc( BLCKSZ * (1+1) ); char* local_blocks_aligned = (char*)TYPEALIGN(PG_IO_ALIGN_SIZE, local_blocks); - // TODO: add check to only encrypt/decrypt tables with specific AM/DB? + RelKeyData* rkd = tde_smgr_get_key(reln); - if(IsCatalogRelationOid(reln->smgr_rlocator.locator.spcOid)) + if(rkd == NULL) { - // Don't try to encrypt catalog tables: - // Issues with bootstrap and encryption metadata mdextend(reln, forknum, blocknum, buffer, skipFsync); return; } int out_len = BLCKSZ; - AesEncrypt(key, iv, ((char*)buffer), BLCKSZ, local_blocks_aligned, &out_len); + AesEncrypt(rkd->internal_key.key, iv, ((char*)buffer), BLCKSZ, local_blocks_aligned, &out_len); mdextend(reln, forknum, blocknum, local_blocks_aligned, skipFsync); @@ -86,10 +139,9 @@ tde_mdreadv(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, mdreadv(reln, forknum, blocknum, buffers, nblocks); - // TODO: add check to only encrypt/decrypt tables with specific AM/DB? + RelKeyData* rkd = tde_smgr_get_key(reln); - // Don't try to decrypt catalog tables, those are not encrypted - if(IsCatalogRelationOid(reln->smgr_rlocator.locator.spcOid)) + if(rkd == NULL) { return; } @@ -113,12 +165,22 @@ tde_mdreadv(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, if(allZero) continue; int out_len = BLCKSZ; - AesDecrypt(key, iv, ((char**)buffers)[i], BLCKSZ, ((char**)buffers)[i], &out_len); + AesDecrypt(rkd->internal_key.key, iv, ((char**)buffers)[i], BLCKSZ, ((char**)buffers)[i], &out_len); } +} + +void +tde_mdcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo) +{ + // This is the only function that gets called during actual CREATE TABLE/INDEX (EVENT TRIGGER) + // so we create the key here by loading it + // Later calls then decide to encrypt or not based on the existence of the key + tde_smgr_get_key(reln); - // And now decrypt buffers in place - // We check the first few bytes of the page: if all zero, we assume it is zero and keep it as is + return mdcreate(reln, forknum, isRedo); } + + static SMgrId tde_smgr_id; static const struct f_smgr tde_smgr = { .name = "tde", @@ -126,7 +188,7 @@ static const struct f_smgr tde_smgr = { .smgr_shutdown = NULL, .smgr_open = mdopen, .smgr_close = mdclose, - .smgr_create = mdcreate, + .smgr_create = tde_mdcreate, .smgr_exists = mdexists, .smgr_unlink = mdunlink, .smgr_extend = tde_mdextend, @@ -152,4 +214,4 @@ void RegisterStorageMgr() void RegisterStorageMgr() { } -#endif \ No newline at end of file +#endif diff --git a/t/results/001_basic.out b/t/results/001_basic.out deleted file mode 100644 index d6f838c98e8..00000000000 --- a/t/results/001_basic.out +++ /dev/null @@ -1,13 +0,0 @@ -CREATE EXTENSION pg_tde; --- server restart -CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING pg_tde; -INSERT INTO test_enc (k) VALUES (5),(6); -SELECT * FROM test_enc ORDER BY id ASC; -1|5 -2|6 --- server restart -SELECT * FROM test_enc ORDER BY id ASC; -1|5 -2|6 -DROP TABLE test_enc; -DROP EXTENSION pg_tde; From dfb9b8d6b358fcb8cba098b1e3dd5daae0d33f7c Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Tue, 18 Jun 2024 12:43:32 +0100 Subject: [PATCH 06/14] Test and other fixes for the smgr branch (#216) With this commit, old pg_tde code should work as before in the same branch, and CI tests also should pass. --- src/access/pg_tde_xlog.c | 6 +++++- src/access/pg_tdeam.c | 4 ++-- src/access/pg_tdeam_handler.c | 6 ++++++ src/catalog/tde_master_key.c | 26 +++++++++++++++++++++++++- src/include/access/pg_tde_xlog.h | 6 ++++++ src/include/access/pg_tdeam.h | 3 +++ src/include/smgr/pg_tde_smgr.h | 2 +- src/pg_tde.c | 6 ++++++ src/pg_tde_event_capture.c | 10 +++++----- src/smgr/pg_tde_smgr.c | 8 ++------ 10 files changed, 61 insertions(+), 16 deletions(-) diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index ae1a746e3ae..b3fd4ec5f74 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -120,6 +120,8 @@ pg_tde_rmgr_identify(uint8 info) return NULL; } +#ifdef PERCONA_FORK + /* * XLog Storage Manager * TODO: @@ -398,4 +400,6 @@ SetXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, char* iv_prefix) iv_prefix[9] = ((lsn >> 16) & 0xFF); iv_prefix[10] = ((lsn >> 8) & 0xFF); iv_prefix[11] = (lsn & 0xFF); -} \ No newline at end of file +} + +#endif diff --git a/src/access/pg_tdeam.c b/src/access/pg_tdeam.c index 7167613f85e..a62dc7f9012 100644 --- a/src/access/pg_tdeam.c +++ b/src/access/pg_tdeam.c @@ -1111,10 +1111,10 @@ pg_tde_getnext(TableScanDesc sscan, ScanDirection direction) * rather than the AM oid, is that this allows to write regression tests * that create another AM reusing the heap handler. */ - if (unlikely(sscan->rs_rd->rd_tableam != GetHeapamTableAmRoutine())) + if (unlikely(sscan->rs_rd->rd_tableam != GetPGTdeamTableAmRoutine())) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg_internal("only heap AM is supported"))); + errmsg_internal("only pg_tde AM is supported"))); /* * We don't expect direct calls to pg_tde_getnext with valid CheckXidAlive diff --git a/src/access/pg_tdeam_handler.c b/src/access/pg_tdeam_handler.c index 0c697b06621..d40fd1d42e9 100644 --- a/src/access/pg_tdeam_handler.c +++ b/src/access/pg_tdeam_handler.c @@ -2623,6 +2623,12 @@ static const TableAmRoutine pg_tdeam_methods = { .scan_sample_next_tuple = pg_tdeam_scan_sample_next_tuple }; +const TableAmRoutine * +GetPGTdeamTableAmRoutine(void) +{ + return &pg_tdeam_methods; +} + Datum pg_tdeam_handler(PG_FUNCTION_ARGS) { diff --git a/src/catalog/tde_master_key.c b/src/catalog/tde_master_key.c index 610f9a6c5e5..618e85deb0e 100644 --- a/src/catalog/tde_master_key.c +++ b/src/catalog/tde_master_key.c @@ -224,12 +224,26 @@ GetMasterKey(Oid dbOid) LWLock *lock_files = tde_lwlock_mk_files(); LWLock *lock_cache = tde_lwlock_mk_cache(); + // TODO: This recursion counter is a dirty hack until the metadata is in the catalog + // As otherwise we would call GetMasterKey recursively and deadlock + static int recursion = 0; + + if(recursion > 0) + { + return NULL; + } + + recursion++; + LWLockAcquire(lock_cache, LW_SHARED); masterKey = get_master_key_from_cache(dbOid); LWLockRelease(lock_cache); if (masterKey) + { + recursion--; return masterKey; + } /* * We should hold an exclusive lock here to ensure that a valid master key, if found, is added @@ -244,6 +258,7 @@ GetMasterKey(Oid dbOid) { LWLockRelease(lock_cache); LWLockRelease(lock_files); + recursion--; return masterKey; } @@ -254,6 +269,7 @@ GetMasterKey(Oid dbOid) LWLockRelease(lock_cache); LWLockRelease(lock_files); + recursion--; return NULL; } @@ -264,6 +280,7 @@ GetMasterKey(Oid dbOid) LWLockRelease(lock_cache); LWLockRelease(lock_files); + recursion--; return NULL; } @@ -273,6 +290,7 @@ GetMasterKey(Oid dbOid) LWLockRelease(lock_cache); LWLockRelease(lock_files); + recursion--; return NULL; } @@ -292,6 +310,7 @@ GetMasterKey(Oid dbOid) if (masterKeyInfo) pfree(masterKeyInfo); + recursion--; return masterKey; } @@ -731,7 +750,12 @@ Datum pg_tde_master_key_info(PG_FUNCTION_ARGS) master_key = GetMasterKey(MyDatabaseId); if (master_key == NULL) - PG_RETURN_NULL(); + { + ereport(ERROR, + (errmsg("Master key does not exists for the database"), + errhint("Use set_master_key interface to set the master key"))); + PG_RETURN_NULL(); + } keyring = GetKeyProviderByID(master_key->keyInfo.keyringId); diff --git a/src/include/access/pg_tde_xlog.h b/src/include/access/pg_tde_xlog.h index 8317fd444a6..f7b37dbe365 100644 --- a/src/include/access/pg_tde_xlog.h +++ b/src/include/access/pg_tde_xlog.h @@ -12,7 +12,9 @@ #include "postgres.h" #include "access/xlog.h" #include "access/xlog_internal.h" +#ifdef PERCONA_FORK #include "access/xlog_smgr.h" +#endif /* TDE XLOG resource manager */ #define XLOG_TDE_ADD_RELATION_KEY 0x00 @@ -35,6 +37,8 @@ static const RmgrData pg_tde_rmgr = { .rm_identify = pg_tde_rmgr_identify }; +#ifdef PERCONA_FORK + /* XLog encryption staff */ /* GUC */ @@ -58,4 +62,6 @@ extern void TDEInitXLogSmgr(void); extern void xlogInitGUC(void); +#endif + #endif /* PG_TDE_XLOG_H */ diff --git a/src/include/access/pg_tdeam.h b/src/include/access/pg_tdeam.h index df3de66170e..13f5795f35c 100644 --- a/src/include/access/pg_tdeam.h +++ b/src/include/access/pg_tdeam.h @@ -333,4 +333,7 @@ extern void HeapCheckForSerializableConflictOut(bool visible, Relation relation, /* Defined in pg_tdeam_handler.c */ extern bool is_pg_tde_rel(Relation rel); +const TableAmRoutine * +GetPGTdeamTableAmRoutine(void); + #endif /* PG_TDEAM_H */ diff --git a/src/include/smgr/pg_tde_smgr.h b/src/include/smgr/pg_tde_smgr.h index 90df06c427f..359b34db35d 100644 --- a/src/include/smgr/pg_tde_smgr.h +++ b/src/include/smgr/pg_tde_smgr.h @@ -1,4 +1,4 @@ #pragma once -extern void RegisterStorageMgr(); \ No newline at end of file +extern void RegisterStorageMgr(); diff --git a/src/pg_tde.c b/src/pg_tde.c index b676b1255c0..563b60ac523 100644 --- a/src/pg_tde.c +++ b/src/pg_tde.c @@ -61,7 +61,9 @@ tde_shmem_request(void) Size sz = TdeRequiredSharedMemorySize(); int required_locks = TdeRequiredLocksCount(); +#ifdef PERCONA_FORK sz = add_size(sz, XLOG_TDE_ENC_BUFF_ALIGNED_SIZE); +#endif if (prev_shmem_request_hook) prev_shmem_request_hook(); @@ -78,8 +80,10 @@ tde_shmem_startup(void) TdeShmemInit(); AesInit(); +#ifdef PERCONA_FORK TDEXLogShmemInit(); TDEInitXLogSmgr(); +#endif } void @@ -92,7 +96,9 @@ _PG_init(void) keyringRegisterVariables(); InitializeMasterKeyInfo(); +#ifdef PERCONA_FORK xlogInitGUC(); +#endif prev_shmem_request_hook = shmem_request_hook; shmem_request_hook = tde_shmem_request; diff --git a/src/pg_tde_event_capture.c b/src/pg_tde_event_capture.c index 3645b45e1ea..d0dc191b1a5 100644 --- a/src/pg_tde_event_capture.c +++ b/src/pg_tde_event_capture.c @@ -52,6 +52,8 @@ GetCurrentTdeCreateEvent(void) Datum pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) { + /* TODO: verify update_compare_indexes failure related to this */ +#ifdef PERCONA_FORK EventTriggerData *trigdata; Node *parsetree; @@ -83,14 +85,10 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) if (rel->rd_rel->relam == get_tde_table_am_oid()) { /* We are creating the index on encrypted table */ - ereport(NOTICE, - (errmsg("Creating index on **ENCRYPTED** relation:%s with Oid:%d", stmt->relation->relname, relationId))); /* set the global state */ tdeCurrentCreateEvent.encryptMode = true; } else - ereport(DEBUG1, - (errmsg("Creating index on relation:%s with Oid:%d", stmt->relation->relname, relationId))); table_close(rel, lockmode); } else @@ -104,12 +102,12 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) tdeCurrentCreateEvent.eventType = TDE_TABLE_CREATE_EVENT; tdeCurrentCreateEvent.relation = stmt->relation; - elog(DEBUG1, "CREATING TABLE %s Using Access Method %s", stmt->relation->relname, stmt->accessMethod); if (stmt->accessMethod && !strcmp(stmt->accessMethod, "pg_tde2")) { tdeCurrentCreateEvent.encryptMode = true; } } +#endif PG_RETURN_NULL(); } @@ -120,6 +118,7 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) Datum pg_tde_ddl_command_end_capture(PG_FUNCTION_ARGS) { +#ifdef PERCONA_FORK /* Ensure this function is being called as an event trigger */ if (!CALLED_AS_EVENT_TRIGGER(fcinfo)) /* internal error */ ereport(ERROR, @@ -134,6 +133,7 @@ pg_tde_ddl_command_end_capture(PG_FUNCTION_ARGS) /* All we need to do is to clear the event state */ reset_current_tde_create_event(); +#endif PG_RETURN_NULL(); } diff --git a/src/smgr/pg_tde_smgr.c b/src/smgr/pg_tde_smgr.c index ceb83cfa186..c6c2400eade 100644 --- a/src/smgr/pg_tde_smgr.c +++ b/src/smgr/pg_tde_smgr.c @@ -8,7 +8,7 @@ #include "access/pg_tde_tdemap.h" #include "pg_tde_event_capture.h" -#if PG_VERSION_NUM >= 170000 +#ifdef PERCONA_FORK // TODO: implement proper IV // iv should be based on blocknum + relfile, available in the API @@ -21,10 +21,6 @@ tde_smgr_get_key(SMgrRelation reln) // As otherwise we would call GetMasterKey recursively and deadlock static int recursion = 0; - ereport(NOTICE, - (errmsg("Trying to decide if table is encrypted: %u", reln->smgr_rlocator.locator.relNumber))); - - if(IsCatalogRelationOid(reln->smgr_rlocator.locator.relNumber)) { // do not try to encrypt/decrypt catalog tables @@ -214,4 +210,4 @@ void RegisterStorageMgr() void RegisterStorageMgr() { } -#endif +#endif /* PERCONA_FORK */ From 2f52b2193f8d0a5f2338f48c093010ddcee0bf63 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Thu, 9 May 2024 18:47:18 +0300 Subject: [PATCH 07/14] Prepare for XLog keyring * Make the *.map *.dat processing code aware of custom databases and table spaces * Add XLog GUC and init the keyring based on that. Only FS for now * Make the internal/external key infrastructure work with custom (not stored in the database) keyrings. --- src/access/pg_tde_tdemap.c | 224 +++++++++++++++++---------- src/access/pg_tde_xlog.c | 149 ++++++++++++++++-- src/catalog/tde_keyring.c | 3 +- src/catalog/tde_master_key.c | 49 +++--- src/include/access/pg_tde_tdemap.h | 22 ++- src/include/access/pg_tde_xlog.h | 1 + src/include/catalog/tde_keyring.h | 7 + src/include/catalog/tde_master_key.h | 6 +- src/transam/pg_tde_xact_handler.c | 2 - 9 files changed, 334 insertions(+), 129 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index ced8f6a26d1..8c4e345f7b7 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -73,10 +73,11 @@ typedef struct TDEMapEntry int32 key_index; } TDEMapEntry; -/* Global variables */ -static char db_path[MAXPGPATH] = {0}; -static char db_map_path[MAXPGPATH] = {0}; -static char db_keydata_path[MAXPGPATH] = {0}; +typedef struct TDEMapFilePath +{ + char map_path[MAXPGPATH]; + char keydata_path[MAXPGPATH]; +} TDEMapFilePath; static void put_key_into_map(Oid rel_id, RelKeyData *key); @@ -84,11 +85,6 @@ static File pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignor static File pg_tde_file_header_write(char *tde_filename, File tde_file, TDEMasterKeyInfo *master_key_info, off_t *bytes_written); static File pg_tde_file_header_read(char *tde_filename, File tde_file, TDEFileHeader *fheader, bool *is_new_file, off_t *bytes_read); -static RelKeyData* tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, TDEMasterKeyInfo *master_key_info); -static RelKeyData *tde_encrypt_rel_key(TDEMasterKey *master_key, RelKeyData *rel_key_data, const RelFileLocator *rlocator); -static RelKeyData *tde_decrypt_rel_key(TDEMasterKey *master_key, RelKeyData *enc_rel_key_data, const RelFileLocator *rlocator); - -static void pg_tde_set_db_file_paths(Oid dbOid); static File pg_tde_open_file(char *tde_filename, TDEMasterKeyInfo *master_key_info, bool should_fill_info, int fileFlags, bool *is_new_file, off_t *offset); static int32 pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMasterKeyInfo *master_key_info); @@ -98,7 +94,7 @@ static bool pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rloca static void pg_tde_write_keydata(char *db_keydata_path, TDEMasterKeyInfo *master_key_info, int32 key_index, RelKeyData *enc_rel_key_data); static void pg_tde_write_one_keydata(File keydata_file, int32 key_index, RelKeyData *enc_rel_key_data); -static RelKeyData* pg_tde_get_key_from_file(const RelFileLocator *rlocator); +static RelKeyData* pg_tde_get_key_from_file(const RelFileLocator *rlocator, GenericKeyring *keyring); static RelKeyData* pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEMasterKey *master_key); static RelKeyData* pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master_key); @@ -140,7 +136,7 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator) } /* Encrypt the key */ - rel_key_data = tde_create_rel_key(newrlocator, &int_key, &master_key->keyInfo); + rel_key_data = tde_create_rel_key(newrlocator->relNumber, &int_key, &master_key->keyInfo); enc_rel_key_data = tde_encrypt_rel_key(master_key, rel_key_data, newrlocator); /* @@ -184,7 +180,29 @@ GetRelationKey(RelFileLocator rel) } } - key = pg_tde_get_key_from_file(&rel); + key = pg_tde_get_key_from_file(&rel, NULL); + + put_key_into_map(rel.relNumber, key); + + return key; +} + +RelKeyData * +GetInternalKey(RelFileLocator rel, GenericKeyring *keyring) +{ + RelKey *curr; + RelKeyData *key; + + Oid rel_id = rel.relNumber; + for (curr = tde_rel_key_map; curr != NULL; curr = curr->next) + { + if (curr->rel_id == rel_id) + { + return curr->key; + } + } + + key = pg_tde_get_key_from_file(&rel, keyring); if (key != NULL) { @@ -226,8 +244,8 @@ tde_sprint_key(InternalKey *k) * Creates a key for a relation identified by rlocator. Returns the newly * created key. */ -static RelKeyData * -tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, TDEMasterKeyInfo *master_key_info) +RelKeyData * +tde_create_rel_key(Oid rel_id, InternalKey *key, TDEMasterKeyInfo *master_key_info) { RelKeyData *rel_key_data; @@ -238,7 +256,7 @@ tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, TDEMasterKe rel_key_data->internal_key.ctx = NULL; /* Add to the decrypted key to cache */ - put_key_into_map(rlocator->relNumber, rel_key_data); + put_key_into_map(rel_id, rel_key_data); return rel_key_data; } @@ -246,7 +264,7 @@ tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, TDEMasterKe /* * Encrypts a given key and returns the encrypted one. */ -static RelKeyData * +RelKeyData * tde_encrypt_rel_key(TDEMasterKey *master_key, RelKeyData *rel_key_data, const RelFileLocator *rlocator) { RelKeyData *enc_rel_key_data; @@ -260,7 +278,7 @@ tde_encrypt_rel_key(TDEMasterKey *master_key, RelKeyData *rel_key_data, const Re /* * Decrypts a given key and returns the decrypted one. */ -static RelKeyData * +RelKeyData * tde_decrypt_rel_key(TDEMasterKey *master_key, RelKeyData *enc_rel_key_data, const RelFileLocator *rlocator) { RelKeyData *rel_key_data = NULL; @@ -271,33 +289,17 @@ tde_decrypt_rel_key(TDEMasterKey *master_key, RelKeyData *enc_rel_key_data, cons return rel_key_data; } -/* - * Sets the global variables so that we don't have to do this again for this - * backend lifetime. - */ -static void -pg_tde_set_db_file_paths(Oid dbOid) +inline void +pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *map_path, char *keydata_path) { - /* Return if the values are already set */ - // TODO: remove this entire global state thing - //if (*db_path && *db_map_path && *db_keydata_path) - // return; - - /* Fill in the values */ - snprintf(db_path, MAXPGPATH, "%s", GetDatabasePath(dbOid, DEFAULTTABLESPACE_OID)); - - /* Set the file nanes for map and keydata */ - join_path_components(db_map_path, db_path, PG_TDE_MAP_FILENAME); - join_path_components(db_keydata_path, db_path, PG_TDE_KEYDATA_FILENAME); -} - -/* - * Path data clean up once the transaction is done. - */ -void -pg_tde_cleanup_path_vars(void) -{ - *db_path = *db_map_path = *db_keydata_path = 0; + if (map_path) + join_path_components(map_path, + GetDatabasePath(rlocator->dbOid, rlocator->spcOid), + PG_TDE_MAP_FILENAME); + if (keydata_path) + join_path_components(keydata_path, + GetDatabasePath(rlocator->dbOid, rlocator->spcOid), + PG_TDE_KEYDATA_FILENAME); } /* @@ -305,10 +307,17 @@ pg_tde_cleanup_path_vars(void) * Returns true if both map and key data files are created. */ void -pg_tde_delete_tde_files(Oid dbOid) +pg_tde_delete_tde_files(Oid dbOid, Oid spcOid) { + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; + /* Set the file paths */ - pg_tde_set_db_file_paths(dbOid); + pg_tde_set_db_file_paths(&(RelFileLocator) { + spcOid, + dbOid, + 0}, + db_map_path, db_keydata_path); /* Remove these files without emitting any error */ PathNameDeleteTemporaryFile(db_map_path, false); @@ -332,9 +341,15 @@ pg_tde_save_master_key(TDEMasterKeyInfo *master_key_info) off_t curr_pos = 0; bool is_new_map = false; bool is_new_key_data = false; + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; /* Set the file paths */ - pg_tde_set_db_file_paths(master_key_info->databaseId); + pg_tde_set_db_file_paths(&(RelFileLocator) { + master_key_info->tablespaceId, + master_key_info->databaseId, + 0}, + db_map_path, db_keydata_path); ereport(LOG, (errmsg("pg_tde_save_master_key"))); @@ -354,16 +369,22 @@ pg_tde_save_master_key(TDEMasterKeyInfo *master_key_info) * a LW_SHARED or higher lock on files before calling this function. */ TDEMasterKeyInfo * -pg_tde_get_master_key(Oid dbOid) +pg_tde_get_master_key(Oid dbOid, Oid spcOid) { File tde_file = -1; TDEFileHeader fheader; TDEMasterKeyInfo *master_key_info = NULL; bool is_new_file = false; off_t bytes_read = 0; + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; /* Set the file paths */ - pg_tde_set_db_file_paths(dbOid); + pg_tde_set_db_file_paths(&(RelFileLocator) { + spcOid, + dbOid, + 0}, + db_map_path, db_keydata_path); /* * Ensuring that we always open the file in binary mode. The caller must @@ -409,7 +430,7 @@ pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignore_missing) tde_file = PathNameOpenFile(tde_filename, fileFlags | PG_BINARY); if (tde_file < 0 && !(errno == ENOENT && ignore_missing == true)) { - ereport(ERROR, + ereport(PANIC, (errcode_for_file_access(), errmsg("Could not open tde file \"%s\": %m", tde_filename))); @@ -594,6 +615,8 @@ pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int fl /* Add the entry to the file */ if (bytes_written != MAP_ENTRY_SIZE) { + char db_map_path[MAXPGPATH] = {0}; + pg_tde_set_db_file_paths(rlocator, db_map_path, NULL); ereport(FATAL, (errcode_for_file_access(), errmsg("Could not write tde map file \"%s\": %m", @@ -772,8 +795,7 @@ pg_tde_write_one_keydata(File keydata_file, int32 key_index, RelKeyData *enc_rel { ereport(FATAL, (errcode_for_file_access(), - errmsg("Could not write tde key data file \"%s\": %m", - db_keydata_path))); + errmsg("Could not write tde key data file: %m"))); } } @@ -823,6 +845,12 @@ pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master /* Check if the file has a valid key */ if ((read_pos + INTERNAL_KEY_LEN) > FileSize(keydata_file)) { + char db_keydata_path[MAXPGPATH] = {0}; + pg_tde_set_db_file_paths(&(RelFileLocator) { + master_key->keyInfo.tablespaceId, + master_key->keyInfo.databaseId, + 0}, + NULL, db_keydata_path); ereport(FATAL, (errcode(ERRCODE_NO_DATA_FOUND), errmsg("Could not find the required key at index %d in tde data file \"%s\": %m", @@ -833,6 +861,12 @@ pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master /* Read the encrypted key */ if (FileRead(keydata_file, &(enc_rel_key_data->internal_key), INTERNAL_KEY_LEN, read_pos, WAIT_EVENT_DATA_FILE_READ) != INTERNAL_KEY_LEN) { + char db_keydata_path[MAXPGPATH] = {0}; + pg_tde_set_db_file_paths(&(RelFileLocator) { + master_key->keyInfo.tablespaceId, + master_key->keyInfo.databaseId, + 0}, + NULL, db_keydata_path); ereport(FATAL, (errcode_for_file_access(), errmsg("Could not read key at index %d in tde key data file \"%s\": %m", @@ -853,13 +887,15 @@ pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_key_data, TDEMasterKeyInfo *master_key_info) { - int32 key_index = 0; - LWLock *lock_files = tde_lwlock_mk_files(); + int32 key_index = 0; + LWLock *lock_files = tde_lwlock_mk_files(); + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; Assert(rlocator); /* Set the file paths */ - pg_tde_set_db_file_paths(rlocator->dbOid); + pg_tde_set_db_file_paths(rlocator, db_map_path, db_keydata_path); /* Create the map entry and then add the encrypted key to the data file */ LWLockAcquire(lock_files, LW_EXCLUSIVE); @@ -877,14 +913,16 @@ pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_k void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator) { - int32 key_index = 0; - off_t offset = 0; - LWLock *lock_files = tde_lwlock_mk_files(); + int32 key_index = 0; + off_t offset = 0; + LWLock *lock_files = tde_lwlock_mk_files(); + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; Assert(rlocator); /* Get the file paths */ - pg_tde_set_db_file_paths(rlocator->dbOid); + pg_tde_set_db_file_paths(rlocator, db_map_path, db_keydata_path); /* Remove the map entry if found */ LWLockAcquire(lock_files, LW_EXCLUSIVE); @@ -920,13 +958,15 @@ pg_tde_delete_key_map_entry(const RelFileLocator *rlocator) void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset) { - int32 key_index = 0; - LWLock *lock_files = tde_lwlock_mk_files(); + int32 key_index = 0; + LWLock *lock_files = tde_lwlock_mk_files(); + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; Assert(rlocator); /* Get the file paths */ - pg_tde_set_db_file_paths(rlocator->dbOid); + pg_tde_set_db_file_paths(rlocator, db_map_path, db_keydata_path); /* Remove the map entry if found */ LWLockAcquire(lock_files, LW_EXCLUSIVE); @@ -950,22 +990,23 @@ pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset) * reads the key data from the keydata file. */ static RelKeyData * -pg_tde_get_key_from_file(const RelFileLocator *rlocator) +pg_tde_get_key_from_file(const RelFileLocator *rlocator, GenericKeyring *keyring) { - int32 key_index = 0; - TDEMasterKey *master_key; - RelKeyData *rel_key_data; - RelKeyData *enc_rel_key_data; - off_t offset = 0; - LWLock *lock_files = tde_lwlock_mk_files(); + int32 key_index = 0; + TDEMasterKey *master_key; + RelKeyData *rel_key_data; + RelKeyData *enc_rel_key_data; + off_t offset = 0; + LWLock *lock_files = tde_lwlock_mk_files(); + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; Assert(rlocator); LWLockAcquire(lock_files, LW_SHARED); /* Get/generate a master, create the key for relation and get the encrypted key with bytes to write */ - pg_tde_set_db_file_paths(rlocator->dbOid); - master_key = GetMasterKey(rlocator->dbOid); + master_key = GetMasterKey(rlocator->dbOid, rlocator->spcOid, keyring); if (master_key == NULL) { LWLockRelease(lock_files); @@ -973,6 +1014,9 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator) (errmsg("failed to retrieve master key"))); } + /* Get the file paths */ + pg_tde_set_db_file_paths(rlocator, db_map_path, db_keydata_path); + /* Read the map entry and get the index of the relation key */ key_index = pg_tde_process_map_entry(rlocator, db_map_path, &offset, false); @@ -1049,12 +1093,18 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key off_t map_size; off_t keydata_size; XLogMasterKeyRotate *xlrec; - off_t xlrec_size; - LWLock *lock_files = tde_lwlock_mk_files(); - LWLock *lock_cache = tde_lwlock_mk_cache(); + off_t xlrec_size; + LWLock *lock_files = tde_lwlock_mk_files(); + LWLock *lock_cache = tde_lwlock_mk_cache(); + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; /* Set the file paths */ - pg_tde_set_db_file_paths(master_key->keyInfo.databaseId); + pg_tde_set_db_file_paths(&(RelFileLocator) { + master_key->keyInfo.tablespaceId, + master_key->keyInfo.databaseId, + 0}, + db_map_path, db_keydata_path); /* Let's update the pathnames in the local variable for ease of use/readability */ strncpy(m_path[OLD_MASTER_KEY], db_map_path, MAXPGPATH); @@ -1158,21 +1208,27 @@ bool pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_size, char *k_file_data) { TDEFileHeader *fheader; - char m_path_new[MAXPGPATH]; - char k_path_new[MAXPGPATH]; - File m_file_new; - File k_file_new; - bool is_new_file; - off_t curr_pos = 0; - off_t read_pos_tmp = 0; - LWLock *lock_files = tde_lwlock_mk_files(); - LWLock *lock_cache = tde_lwlock_mk_cache(); + char m_path_new[MAXPGPATH]; + char k_path_new[MAXPGPATH]; + File m_file_new; + File k_file_new; + bool is_new_file; + off_t curr_pos = 0; + off_t read_pos_tmp = 0; + LWLock *lock_files = tde_lwlock_mk_files(); + LWLock *lock_cache = tde_lwlock_mk_cache(); + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; /* Let's get the header. Buff should start with the map file header. */ fheader = (TDEFileHeader *) m_file_data; /* Set the file paths */ - pg_tde_set_db_file_paths(fheader->master_key_info.databaseId); + pg_tde_set_db_file_paths(&(RelFileLocator) { + fheader->master_key_info.tablespaceId, + fheader->master_key_info.databaseId, + 0}, + db_map_path, db_keydata_path); LWLockAcquire(lock_files, LW_EXCLUSIVE); LWLockAcquire(lock_cache, LW_EXCLUSIVE); diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index b3fd4ec5f74..bdbbb553c7f 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -16,6 +16,7 @@ #include "access/xlog.h" #include "access/xlog_internal.h" #include "access/xloginsert.h" +#include "catalog/pg_tablespace_d.h" #include "storage/bufmgr.h" #include "storage/shmem.h" #include "utils/guc.h" @@ -23,6 +24,7 @@ #include "access/pg_tde_tdemap.h" #include "access/pg_tde_xlog.h" +#include "catalog/tde_keyring.h" #include "catalog/tde_master_key.h" #include "encryption/enc_tde.h" @@ -30,12 +32,36 @@ static char *TDEXLogEncryptBuf = NULL; bool EncryptXLog = false; +/* GUC */ +static char *KRingProviderType = NULL; +static char *KRingProviderFilePath = NULL; + static XLogPageHeaderData EncryptCurrentPageHrd; static XLogPageHeaderData DecryptCurrentPageHrd; static ssize_t TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset); static void SetXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, char* iv_prefix); static int XLOGChooseNumBuffers(void); + +typedef enum +{ + TDE_GCAT_KEY_XLOG, + + /* must be last */ + TDE_GCAT_KEYS_COUNT +} GlobalCatalogKeyTypes; + +/* TODO: move TDEXLogEncryptBuf here*/ +typedef struct XLogEncryptionState +{ + GenericKeyring *keyring; + /* TODO: locking */ + TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT]; + +} XLogEncryptionState; + +static XLogEncryptionState *EncryptionState = NULL; + /* * TDE fork XLog */ @@ -123,16 +149,54 @@ pg_tde_rmgr_identify(uint8 info) #ifdef PERCONA_FORK /* + * ------------------------- * XLog Storage Manager - * TODO: - * - Should be a config option "on/off"? - * - Currently it encrypts WAL XLog Pages, should we encrypt whole Segments? `initdb` for - * example generates a write of 312 pages - so 312 "gen IV" and "encrypt" runs instead of one. - * Would require though an extra read() during recovery/was_send etc to check `XLogPageHeader` - * if segment is encrypted. - * We could also encrypt Records while adding them to the XLog Buf but it'll be the slowest (?). */ +static GenericKeyring *xlog_keyring; + +static void +pg_tde_init_xlog_kring(void) +{ + EncryptionState->keyring->type = get_keyring_provider_from_typename(KRingProviderType); + switch (EncryptionState->keyring->type) + { + case FILE_KEY_PROVIDER: + FileKeyring *kring = (FileKeyring *) EncryptionState->keyring; + strncpy(kring->file_name, KRingProviderFilePath, sizeof(kring->file_name)); + break; + } +} + +static void +pg_tde_create_xlog_key(void) +{ + InternalKey int_key; + RelKeyData *rel_key_data; + RelKeyData *enc_rel_key_data; + RelFileLocator *rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); + TDEMasterKey *master_key; + + master_key = set_master_key_with_keyring("xlog-master-key", xlog_keyring, + rlocator->dbOid, rlocator->spcOid, false); + + memset(&int_key, 0, sizeof(InternalKey)); + + if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) + { + ereport(FATAL, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate internal key for \"WAL\": %s", + ERR_error_string(ERR_get_error(), NULL)))); + } + + rel_key_data = tde_create_rel_key(rlocator->relNumber, &int_key, &master_key->keyInfo); + enc_rel_key_data = tde_encrypt_rel_key(master_key, rel_key_data, rlocator); + + pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &master_key->keyInfo); + memcpy(EncryptionState->master_keys + TDE_GCAT_KEY_XLOG, master_key, sizeof(TDEMasterKey)); +} + void xlogInitGUC(void) { @@ -147,6 +211,28 @@ xlogInitGUC(void) NULL, /* assign_hook */ NULL /* show_hook */ ); + DefineCustomStringVariable("pg_tde.wal_keyring_type", + "Keyring type for XLog", + NULL, + &KRingProviderType, + NULL, + PGC_POSTMASTER, + 0, /* no flags required */ + NULL, + NULL, + NULL + ); + DefineCustomStringVariable("pg_tde.wal_keyring_file_path", + "Keyring file options for XLog", + NULL, + &KRingProviderFilePath, + NULL, + PGC_POSTMASTER, + 0, /* no flags required */ + NULL, + NULL, + NULL + ); } static int @@ -174,6 +260,17 @@ TDEXLogEncryptBuffSize() return (Size) XLOG_BLCKSZ * xbuffers; } +Size +XLogEncStateSize() +{ + Size size; + + size = sizeof(XLogEncryptionState); + size = add_size(size, sizeof(KeyringProviders)); + + return MAXALIGN(size); +} + /* * Alloc memory for the encryption buffer. * @@ -187,10 +284,11 @@ TDEXLogEncryptBuffSize() void TDEXLogShmemInit(void) { + bool foundBuf; + char *allocptr; + if (EncryptXLog) { - bool foundBuf; - TDEXLogEncryptBuf = (char *) TYPEALIGN(PG_IO_ALIGN_SIZE, ShmemInitStruct("TDE XLog Encryption Buffer", @@ -199,12 +297,21 @@ TDEXLogShmemInit(void) elog(DEBUG1, "pg_tde: initialized encryption buffer %lu bytes", XLOG_TDE_ENC_BUFF_ALIGNED_SIZE); } + + EncryptionState = (XLogEncryptionState *) + ShmemInitStruct("TDE XLog Encryption State", + XLogEncStateSize(), &foundBuf); + + allocptr = ((char *) EncryptionState) + MAXALIGN(sizeof(XLogEncryptionState)); + EncryptionState->keyring = allocptr; } void TDEInitXLogSmgr(void) { SetXLogSmgr(&tde_xlog_smgr); + pg_tde_init_xlog_kring(); + pg_tde_create_xlog_key(); } /* @@ -232,11 +339,15 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset) size_t data_size = 0; XLogPageHeader curr_page_hdr = &EncryptCurrentPageHrd; XLogPageHeader enc_buf_page; - RelKeyData key = {.internal_key = XLogInternalKey}; + // RelKeyData key = {.internal_key = XLogInternalKey}; + RelKeyData *key = NULL; off_t enc_off; size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; uint32 iv_ctr = 0; + pg_tde_init_xlog_kring(); + key = GetInternalKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), xlog_keyring); + #ifdef TDE_XLOG_DEBUG elog(DEBUG1, "write encrypted WAL, pages amount: %d, size: %lu offset: %ld", count / (Size) XLOG_BLCKSZ, count, offset); @@ -300,7 +411,7 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset) { SetXLogPageIVPrefix(curr_page_hdr->xlp_tli, curr_page_hdr->xlp_pageaddr, iv_prefix); PG_TDE_ENCRYPT_DATA(iv_prefix, iv_ctr, (char *) buf + enc_off, data_size, - TDEXLogEncryptBuf + enc_off, &key); + TDEXLogEncryptBuf + enc_off, key); } page_size = XLOG_BLCKSZ; @@ -320,7 +431,8 @@ pg_tde_xlog_seg_read(int fd, void *buf, size_t count, off_t offset) char iv_prefix[16] = {0,}; size_t data_size = 0; XLogPageHeader curr_page_hdr = &DecryptCurrentPageHrd; - RelKeyData key = {.internal_key = XLogInternalKey}; + // RelKeyData key = {.internal_key = XLogInternalKey}; + RelKeyData *key = NULL; size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; off_t dec_off; uint32 iv_ctr = 0; @@ -329,6 +441,17 @@ pg_tde_xlog_seg_read(int fd, void *buf, size_t count, off_t offset) elog(DEBUG1, "read from a WAL segment, pages amount: %d, size: %lu offset: %ld", count / (Size) XLOG_BLCKSZ, count, offset); #endif + pg_tde_init_xlog_kring(); + { + char db_map_path[MAXPGPATH] = {0}; + + pg_tde_set_db_file_paths(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), + db_map_path, NULL); + if (access(db_map_path, F_OK) == -1) + pg_tde_create_xlog_key(); + } + key = GetInternalKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), xlog_keyring); + readsz = pg_pread(fd, buf, count, offset); /* @@ -373,7 +496,7 @@ pg_tde_xlog_seg_read(int fd, void *buf, size_t count, off_t offset) SetXLogPageIVPrefix(curr_page_hdr->xlp_tli, curr_page_hdr->xlp_pageaddr, iv_prefix); PG_TDE_DECRYPT_DATA( iv_prefix, iv_ctr, - (char *) buf + dec_off, data_size, (char *) buf + dec_off, &key); + (char *) buf + dec_off, data_size, (char *) buf + dec_off, key); } page_size = XLOG_BLCKSZ; diff --git a/src/catalog/tde_keyring.c b/src/catalog/tde_keyring.c index fc56deee7ba..5397dcb7a90 100644 --- a/src/catalog/tde_keyring.c +++ b/src/catalog/tde_keyring.c @@ -50,13 +50,12 @@ PG_FUNCTION_INFO_V1(keyring_delete_dependency_check_trigger); #define FILE_KEYRING_TYPE_KEY "type" static FileKeyring *load_file_keyring_provider_options(Datum keyring_options); -static ProviderType get_keyring_provider_from_typename(char *provider_type); static GenericKeyring *load_keyring_provider_options(ProviderType provider_type, Datum keyring_options); static VaultV2Keyring *load_vaultV2_keyring_provider_options(Datum keyring_options); static void debug_print_kerying(GenericKeyring *keyring); static GenericKeyring *load_keyring_provider_from_tuple(HeapTuple tuple, TupleDesc tupDesc); -static ProviderType +ProviderType get_keyring_provider_from_typename(char *provider_type) { if (provider_type == NULL) diff --git a/src/catalog/tde_master_key.c b/src/catalog/tde_master_key.c index 618e85deb0e..e1b5d913b20 100644 --- a/src/catalog/tde_master_key.c +++ b/src/catalog/tde_master_key.c @@ -72,7 +72,6 @@ static void clear_master_key_cache(Oid databaseId, Oid tablespaceId) ; static inline dshash_table *get_master_key_Hash(void); static TDEMasterKey *get_master_key_from_cache(Oid dbOid); static void push_master_key_to_cache(TDEMasterKey *masterKey); -static TDEMasterKey *set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, bool ensure_new_key); static const TDEShmemSetupRoutine master_key_info_shmem_routine = { .init_shared_state = initialize_shared_state, @@ -214,11 +213,10 @@ save_master_key_info(TDEMasterKeyInfo *master_key_info) * throws an error. */ TDEMasterKey * -GetMasterKey(Oid dbOid) +GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) { TDEMasterKey *masterKey = NULL; TDEMasterKeyInfo *masterKeyInfo = NULL; - GenericKeyring *keyring = NULL; const keyInfo *keyInfo = NULL; KeyringReturnCodes keyring_ret; LWLock *lock_files = tde_lwlock_mk_files(); @@ -263,7 +261,7 @@ GetMasterKey(Oid dbOid) } /* Master key not present in cache. Load from the keyring */ - masterKeyInfo = pg_tde_get_master_key(dbOid); + masterKeyInfo = pg_tde_get_master_key(dbOid, spcOid); if (masterKeyInfo == NULL) { LWLockRelease(lock_cache); @@ -273,12 +271,12 @@ GetMasterKey(Oid dbOid) return NULL; } - /* Load the master key from keyring and store it in cache */ - keyring = GetKeyProviderByID(masterKeyInfo->keyringId); - if (keyring == NULL) - { - LWLockRelease(lock_cache); - LWLockRelease(lock_files); + if (keyring == NULL) { + keyring = GetKeyProviderByID(masterKeyInfo->keyringId); + if (keyring == NULL) + { + LWLockRelease(lock_cache); + LWLockRelease(lock_files); recursion--; return NULL; @@ -324,12 +322,11 @@ GetMasterKey(Oid dbOid) * to make sure if some other caller has not added a master key for * same database while we were waiting for the lock. */ - -static TDEMasterKey * -set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, bool ensure_new_key) +TDEMasterKey * +set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, + Oid dbOid, Oid spcOid, bool ensure_new_key) { TDEMasterKey *masterKey = NULL; - Oid dbOid = MyDatabaseId; LWLock *lock_files = tde_lwlock_mk_files(); LWLock *lock_cache = tde_lwlock_mk_cache(); bool is_dup_key = false; @@ -345,14 +342,15 @@ set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, bool /* TODO: Add the key in the cache? */ if (is_dup_key == false) - is_dup_key = (pg_tde_get_master_key(dbOid) != NULL); + is_dup_key = (pg_tde_get_master_key(dbOid, spcOid) != NULL); if (is_dup_key == false) { const keyInfo *keyInfo = NULL; masterKey = palloc(sizeof(TDEMasterKey)); - masterKey->keyInfo.databaseId = MyDatabaseId; + masterKey->keyInfo.databaseId = dbOid; + masterKey->keyInfo.tablespaceId = spcOid; masterKey->keyInfo.keyId.version = DEFAULT_MASTER_KEY_VERSION; masterKey->keyInfo.keyringId = keyring->key_id; strncpy(masterKey->keyInfo.keyId.name, key_name, TDE_KEY_NAME_LEN); @@ -378,9 +376,9 @@ set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, bool save_master_key_info(&masterKey->keyInfo); /* XLog the new key*/ - XLogBeginInsert(); - XLogRegisterData((char *) &masterKey->keyInfo, sizeof(TDEMasterKeyInfo)); - XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_MASTER_KEY); + // XLogBeginInsert(); + // XLogRegisterData((char *) &masterKey->keyInfo, sizeof(TDEMasterKeyInfo)); + // XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_MASTER_KEY); push_master_key_to_cache(masterKey); } @@ -407,7 +405,10 @@ set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, bool bool SetMasterKey(const char *key_name, const char *provider_name, bool ensure_new_key) { - TDEMasterKey *master_key = set_master_key_with_keyring(key_name, GetKeyProviderByName(provider_name), ensure_new_key); + TDEMasterKey *master_key = set_master_key_with_keyring(key_name, + GetKeyProviderByName(provider_name), + MyDatabaseId, MyDatabaseTableSpace, + ensure_new_key); return (master_key != NULL); } @@ -415,7 +416,7 @@ SetMasterKey(const char *key_name, const char *provider_name, bool ensure_new_ke bool RotateMasterKey(const char *new_key_name, const char *new_provider_name, bool ensure_new_key) { - TDEMasterKey *master_key = GetMasterKey(MyDatabaseId); + TDEMasterKey *master_key = GetMasterKey(MyDatabaseId, MyDatabaseTableSpace, NULL); TDEMasterKey new_master_key; const keyInfo *keyInfo = NULL; GenericKeyring *keyring; @@ -564,7 +565,7 @@ GetMasterKeyProviderId(void) } { /* Master key not present in cache. Try Loading it from the info file */ - masterKeyInfo = pg_tde_get_master_key(dbOid); + masterKeyInfo = pg_tde_get_master_key(dbOid, MyDatabaseTableSpace); if (masterKeyInfo) { keyringId = masterKeyInfo->keyringId; @@ -671,7 +672,7 @@ cleanup_master_key_info(Oid databaseId, Oid tablespaceId) */ /* Remove the tde files */ - pg_tde_delete_tde_files(databaseId); + pg_tde_delete_tde_files(databaseId, tablespaceId); } static void @@ -748,7 +749,7 @@ Datum pg_tde_master_key_info(PG_FUNCTION_ARGS) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context that cannot accept type record"))); - master_key = GetMasterKey(MyDatabaseId); + master_key = GetMasterKey(MyDatabaseId, MyDatabaseTableSpace, NULL); if (master_key == NULL) { ereport(ERROR, diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index a0b4a899e2a..4b72bf5efb1 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -10,10 +10,21 @@ #include "utils/rel.h" #include "access/xlog_internal.h" +#include "catalog/pg_tablespace_d.h" #include "catalog/tde_master_key.h" #include "storage/fd.h" #include "storage/relfilelocator.h" +/* + * Neeed for glogbal data (WAL etc) keys identification in caches and storage. + * We take IDs the oid type operators, so there is no overlap with the "real" + * catalog object possible. + */ +#define GLOBAL_DATA_TDE_OID 607 /* Global objects fake "db" */ +#define XLOG_TDE_OID 608 + +#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) {GLOBALTABLESPACE_OID, 0, _obj_oid} + typedef struct InternalKey { uint8 key[INTERNAL_KEY_LEN]; @@ -52,14 +63,19 @@ extern void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator); extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset); extern RelKeyData *GetRelationKey(RelFileLocator rel); +extern RelKeyData *GetInternalKey(RelFileLocator rel, GenericKeyring *keyring); -extern void pg_tde_cleanup_path_vars(void); -extern void pg_tde_delete_tde_files(Oid dbOid); +extern void pg_tde_delete_tde_files(Oid dbOid, Oid spcOid); -extern TDEMasterKeyInfo *pg_tde_get_master_key(Oid dbOid); +extern TDEMasterKeyInfo *pg_tde_get_master_key(Oid dbOid, Oid spcOid); extern bool pg_tde_save_master_key(TDEMasterKeyInfo *master_key_info); extern bool pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key); extern bool pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_size, char *k_file_data); +extern RelKeyData* tde_create_rel_key(Oid rel_id, InternalKey *key, TDEMasterKeyInfo *master_key_info); +extern RelKeyData *tde_encrypt_rel_key(TDEMasterKey *master_key, RelKeyData *rel_key_data, const RelFileLocator *rlocator); +extern RelKeyData *tde_decrypt_rel_key(TDEMasterKey *master_key, RelKeyData *enc_rel_key_data, const RelFileLocator *rlocator); + +extern void pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *map_path, char *keydata_path); const char * tde_sprint_key(InternalKey *k); diff --git a/src/include/access/pg_tde_xlog.h b/src/include/access/pg_tde_xlog.h index f7b37dbe365..c380b8b207f 100644 --- a/src/include/access/pg_tde_xlog.h +++ b/src/include/access/pg_tde_xlog.h @@ -45,6 +45,7 @@ static const RmgrData pg_tde_rmgr = { extern bool EncryptXLog; extern Size TDEXLogEncryptBuffSize(); +extern Size XLogEncStateSize(); #define XLOG_TDE_ENC_BUFF_ALIGNED_SIZE add_size(TDEXLogEncryptBuffSize(), PG_IO_ALIGN_SIZE) diff --git a/src/include/catalog/tde_keyring.h b/src/include/catalog/tde_keyring.h index 29b2100e5a2..5eb310cdf13 100644 --- a/src/include/catalog/tde_keyring.h +++ b/src/include/catalog/tde_keyring.h @@ -54,8 +54,15 @@ typedef struct VaultV2Keyring char vault_mount_path[MAXPGPATH]; } VaultV2Keyring; +typedef union KeyringProviders +{ + FileKeyring file; + VaultV2Keyring vault; +} KeyringProviders; + extern List *GetAllKeyringProviders(void); extern GenericKeyring *GetKeyProviderByName(const char *provider_name); extern GenericKeyring *GetKeyProviderByID(int provider_id); +extern ProviderType get_keyring_provider_from_typename(char *provider_type); #endif /*TDE_KEYRING_H*/ diff --git a/src/include/catalog/tde_master_key.h b/src/include/catalog/tde_master_key.h index 77bad07f11c..a6f82123ead 100644 --- a/src/include/catalog/tde_master_key.h +++ b/src/include/catalog/tde_master_key.h @@ -68,9 +68,13 @@ extern LWLock *tde_lwlock_mk_cache(void); extern bool save_master_key_info(TDEMasterKeyInfo *masterKeyInfo); extern Oid GetMasterKeyProviderId(void); -extern TDEMasterKey* GetMasterKey(Oid); +extern TDEMasterKey* GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring); extern bool SetMasterKey(const char *key_name, const char *provider_name, bool ensure_new_key); extern bool RotateMasterKey(const char *new_key_name, const char *new_provider_name, bool ensure_new_key); extern bool xl_tde_perform_rotate_key(XLogMasterKeyRotate *xlrec); +extern TDEMasterKey *set_master_key_with_keyring(const char *key_name, + GenericKeyring *keyring, + Oid dbOid, Oid spcOid, + bool ensure_new_key); #endif /*PG_TDE_MASTER_KEY_H*/ diff --git a/src/transam/pg_tde_xact_handler.c b/src/transam/pg_tde_xact_handler.c index ff84c3d91e4..4b0576a0e02 100644 --- a/src/transam/pg_tde_xact_handler.c +++ b/src/transam/pg_tde_xact_handler.c @@ -52,8 +52,6 @@ pg_tde_xact_callback(XactEvent event, void *arg) { pending_delete_cleanup(); } - - pg_tde_cleanup_path_vars(); } void From 594a582dc43a2b34e1d579976a85b9234263d4c1 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Thu, 16 May 2024 18:39:59 +0300 Subject: [PATCH 08/14] Create and use global catalog key * Check and create an internal key for XLog during the server start. If the key is created (not the first start with the EncryptWAL), then upload it into the cache. We can't read the key from files while writing the XLog to the disk as it happens in the critical section and no palloc is allowed. * Create a custom cache for the global catalog external key as we can't use PG's hashmap during the (again, no pallocs in critical section). --- Makefile.in | 1 + meson.build | 1 + src/access/pg_tde_tdemap.c | 19 +- src/access/pg_tde_xlog.c | 133 +------------- src/catalog/tde_global_catalog.c | 225 +++++++++++++++++++++++ src/catalog/tde_master_key.c | 25 ++- src/include/access/pg_tde_tdemap.h | 10 - src/include/access/pg_tde_xlog.h | 14 +- src/include/catalog/tde_global_catalog.h | 37 ++++ src/include/catalog/tde_master_key.h | 6 +- src/keyring/.keyring_api.c.swp | Bin 12288 -> 0 bytes src/pg_tde.c | 14 +- 12 files changed, 321 insertions(+), 164 deletions(-) create mode 100644 src/catalog/tde_global_catalog.c create mode 100644 src/include/catalog/tde_global_catalog.h delete mode 100644 src/keyring/.keyring_api.c.swp diff --git a/Makefile.in b/Makefile.in index 8f530f2dc7f..7671467a504 100644 --- a/Makefile.in +++ b/Makefile.in @@ -40,6 +40,7 @@ src/keyring/keyring_curl.o \ src/keyring/keyring_file.o \ src/keyring/keyring_vault.o \ src/keyring/keyring_api.o \ +src/catalog/tde_global_catalog.o \ src/catalog/tde_keyring.o \ src/catalog/tde_master_key.o \ src/common/pg_tde_shmem.o \ diff --git a/meson.build b/meson.build index b9cb1e87184..01ba44b4fe5 100644 --- a/meson.build +++ b/meson.build @@ -40,6 +40,7 @@ pg_tde_sources = files( 'src/smgr/pg_tde_smgr.c', + 'src/catalog/tde_global_catalog.c', 'src/catalog/tde_keyring.c', 'src/catalog/tde_master_key.c', 'src/common/pg_tde_shmem.c', diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 8c4e345f7b7..13ac03d7d50 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -292,14 +292,21 @@ tde_decrypt_rel_key(TDEMasterKey *master_key, RelKeyData *enc_rel_key_data, cons inline void pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *map_path, char *keydata_path) { + char *db_path; + + /* We use dbOid for the global space for key caches but for the backend + * it should be 0. + */ + if (rlocator->spcOid == GLOBALTABLESPACE_OID) + db_path = GetDatabasePath(0, rlocator->spcOid); + else + db_path = GetDatabasePath(rlocator->dbOid, rlocator->spcOid); + + if (map_path) - join_path_components(map_path, - GetDatabasePath(rlocator->dbOid, rlocator->spcOid), - PG_TDE_MAP_FILENAME); + join_path_components(map_path, db_path, PG_TDE_MAP_FILENAME); if (keydata_path) - join_path_components(keydata_path, - GetDatabasePath(rlocator->dbOid, rlocator->spcOid), - PG_TDE_KEYDATA_FILENAME); + join_path_components(keydata_path, db_path, PG_TDE_KEYDATA_FILENAME); } /* diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index bdbbb553c7f..c3c661b6be1 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -24,17 +24,14 @@ #include "access/pg_tde_tdemap.h" #include "access/pg_tde_xlog.h" -#include "catalog/tde_keyring.h" -#include "catalog/tde_master_key.h" +#include "catalog/tde_global_catalog.h" #include "encryption/enc_tde.h" static char *TDEXLogEncryptBuf = NULL; -bool EncryptXLog = false; /* GUC */ -static char *KRingProviderType = NULL; -static char *KRingProviderFilePath = NULL; +static bool EncryptXLog = false; static XLogPageHeaderData EncryptCurrentPageHrd; static XLogPageHeaderData DecryptCurrentPageHrd; @@ -52,15 +49,13 @@ typedef enum } GlobalCatalogKeyTypes; /* TODO: move TDEXLogEncryptBuf here*/ -typedef struct XLogEncryptionState +typedef struct EncryptionStateData { GenericKeyring *keyring; /* TODO: locking */ TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT]; -} XLogEncryptionState; - -static XLogEncryptionState *EncryptionState = NULL; +} EncryptionStateData; /* * TDE fork XLog @@ -153,52 +148,8 @@ pg_tde_rmgr_identify(uint8 info) * XLog Storage Manager */ -static GenericKeyring *xlog_keyring; - -static void -pg_tde_init_xlog_kring(void) -{ - EncryptionState->keyring->type = get_keyring_provider_from_typename(KRingProviderType); - switch (EncryptionState->keyring->type) - { - case FILE_KEY_PROVIDER: - FileKeyring *kring = (FileKeyring *) EncryptionState->keyring; - strncpy(kring->file_name, KRingProviderFilePath, sizeof(kring->file_name)); - break; - } -} - -static void -pg_tde_create_xlog_key(void) -{ - InternalKey int_key; - RelKeyData *rel_key_data; - RelKeyData *enc_rel_key_data; - RelFileLocator *rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); - TDEMasterKey *master_key; - - master_key = set_master_key_with_keyring("xlog-master-key", xlog_keyring, - rlocator->dbOid, rlocator->spcOid, false); - - memset(&int_key, 0, sizeof(InternalKey)); - - if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) - { - ereport(FATAL, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not generate internal key for \"WAL\": %s", - ERR_error_string(ERR_get_error(), NULL)))); - } - - rel_key_data = tde_create_rel_key(rlocator->relNumber, &int_key, &master_key->keyInfo); - enc_rel_key_data = tde_encrypt_rel_key(master_key, rel_key_data, rlocator); - - pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &master_key->keyInfo); - memcpy(EncryptionState->master_keys + TDE_GCAT_KEY_XLOG, master_key, sizeof(TDEMasterKey)); -} - void -xlogInitGUC(void) +XLogInitGUC(void) { DefineCustomBoolVariable("pg_tde.wal_encrypt", /* name */ "Enable/Disable encryption of WAL.", /* short_desc */ @@ -211,28 +162,6 @@ xlogInitGUC(void) NULL, /* assign_hook */ NULL /* show_hook */ ); - DefineCustomStringVariable("pg_tde.wal_keyring_type", - "Keyring type for XLog", - NULL, - &KRingProviderType, - NULL, - PGC_POSTMASTER, - 0, /* no flags required */ - NULL, - NULL, - NULL - ); - DefineCustomStringVariable("pg_tde.wal_keyring_file_path", - "Keyring file options for XLog", - NULL, - &KRingProviderFilePath, - NULL, - PGC_POSTMASTER, - 0, /* no flags required */ - NULL, - NULL, - NULL - ); } static int @@ -252,7 +181,7 @@ XLOGChooseNumBuffers(void) * Defines the size of the XLog encryption buffer */ Size -TDEXLogEncryptBuffSize() +TDEXLogEncryptBuffSize(void) { int xbuffers; @@ -260,17 +189,6 @@ TDEXLogEncryptBuffSize() return (Size) XLOG_BLCKSZ * xbuffers; } -Size -XLogEncStateSize() -{ - Size size; - - size = sizeof(XLogEncryptionState); - size = add_size(size, sizeof(KeyringProviders)); - - return MAXALIGN(size); -} - /* * Alloc memory for the encryption buffer. * @@ -285,7 +203,6 @@ void TDEXLogShmemInit(void) { bool foundBuf; - char *allocptr; if (EncryptXLog) { @@ -297,29 +214,14 @@ TDEXLogShmemInit(void) elog(DEBUG1, "pg_tde: initialized encryption buffer %lu bytes", XLOG_TDE_ENC_BUFF_ALIGNED_SIZE); } - - EncryptionState = (XLogEncryptionState *) - ShmemInitStruct("TDE XLog Encryption State", - XLogEncStateSize(), &foundBuf); - - allocptr = ((char *) EncryptionState) + MAXALIGN(sizeof(XLogEncryptionState)); - EncryptionState->keyring = allocptr; } void -TDEInitXLogSmgr(void) +TDEXLogSmgrInit(void) { SetXLogSmgr(&tde_xlog_smgr); - pg_tde_init_xlog_kring(); - pg_tde_create_xlog_key(); } -/* - * TODO: proper key management - * where to store refs to the master and internal keys? - */ -static InternalKey XLogInternalKey = {.key = {0xD,}}; - ssize_t pg_tde_xlog_seg_write(int fd, const void *buf, size_t count, off_t offset) { @@ -339,16 +241,11 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset) size_t data_size = 0; XLogPageHeader curr_page_hdr = &EncryptCurrentPageHrd; XLogPageHeader enc_buf_page; - // RelKeyData key = {.internal_key = XLogInternalKey}; - RelKeyData *key = NULL; + RelKeyData *key = GetGlCatInternalKey(XLOG_TDE_OID); off_t enc_off; size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; uint32 iv_ctr = 0; - pg_tde_init_xlog_kring(); - key = GetInternalKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), xlog_keyring); - - #ifdef TDE_XLOG_DEBUG elog(DEBUG1, "write encrypted WAL, pages amount: %d, size: %lu offset: %ld", count / (Size) XLOG_BLCKSZ, count, offset); #endif @@ -431,8 +328,7 @@ pg_tde_xlog_seg_read(int fd, void *buf, size_t count, off_t offset) char iv_prefix[16] = {0,}; size_t data_size = 0; XLogPageHeader curr_page_hdr = &DecryptCurrentPageHrd; - // RelKeyData key = {.internal_key = XLogInternalKey}; - RelKeyData *key = NULL; + RelKeyData *key = GetGlCatInternalKey(XLOG_TDE_OID); size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; off_t dec_off; uint32 iv_ctr = 0; @@ -441,17 +337,6 @@ pg_tde_xlog_seg_read(int fd, void *buf, size_t count, off_t offset) elog(DEBUG1, "read from a WAL segment, pages amount: %d, size: %lu offset: %ld", count / (Size) XLOG_BLCKSZ, count, offset); #endif - pg_tde_init_xlog_kring(); - { - char db_map_path[MAXPGPATH] = {0}; - - pg_tde_set_db_file_paths(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), - db_map_path, NULL); - if (access(db_map_path, F_OK) == -1) - pg_tde_create_xlog_key(); - } - key = GetInternalKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), xlog_keyring); - readsz = pg_pread(fd, buf, count, offset); /* diff --git a/src/catalog/tde_global_catalog.c b/src/catalog/tde_global_catalog.c new file mode 100644 index 00000000000..a6c694181fc --- /dev/null +++ b/src/catalog/tde_global_catalog.c @@ -0,0 +1,225 @@ +/*------------------------------------------------------------------------- + * + * tde_global_catalog.c + * Global catalog key management + * + * + * IDENTIFICATION + * src/catalog/tde_global_catalog.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "storage/shmem.h" +#include "utils/guc.h" + +#include "access/pg_tde_tdemap.h" +#include "catalog/tde_global_catalog.h" +#include "catalog/tde_keyring.h" +#include "catalog/tde_master_key.h" + +#include +#include +#include + +typedef enum +{ + TDE_GCAT_KEY_XLOG, + + /* must be last */ + TDE_GCAT_KEYS_COUNT +} GlobalCatalogKeyTypes; + +typedef struct EncryptionStateData +{ + GenericKeyring *keyring; + TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT]; +} EncryptionStateData; + +static EncryptionStateData *EncryptionState = NULL; + +/* GUC */ +static char *KRingProviderType = NULL; +static char *KRingProviderFilePath = NULL; + +static void init_gl_catalog_keys(void); +static void init_keyring(void); +static TDEMasterKey *create_master_key(const char *key_name, + GenericKeyring *keyring, Oid dbOid, Oid spcOid, + bool ensure_new_key); + +void +TDEGlCatInitGUC(void) +{ + DefineCustomStringVariable("pg_tde.global_keyring_type", + "Keyring type for global catalog", + NULL, + &KRingProviderType, + NULL, + PGC_POSTMASTER, + 0, /* no flags required */ + NULL, + NULL, + NULL + ); + DefineCustomStringVariable("pg_tde.global_keyring_file_path", + "Keyring file options for global catalog", + NULL, + &KRingProviderFilePath, + NULL, + PGC_POSTMASTER, + 0, /* no flags required */ + NULL, + NULL, + NULL + ); +} + + +Size +TDEGlCatEncStateSize(void) +{ + Size size; + + size = sizeof(EncryptionStateData); + size = add_size(size, sizeof(KeyringProviders)); + + return MAXALIGN(size); +} + +void +TDEGlCatShmemInit(void) +{ + bool foundBuf; + char *allocptr; + + EncryptionState = (EncryptionStateData *) + ShmemInitStruct("TDE XLog Encryption State", + TDEGlCatEncStateSize(), &foundBuf); + + allocptr = ((char *) EncryptionState) + MAXALIGN(sizeof(EncryptionStateData)); + EncryptionState->keyring = (GenericKeyring *) allocptr; + memset(EncryptionState->keyring, 0, sizeof(KeyringProviders)); + memset(EncryptionState->master_keys, 0, sizeof(TDEMasterKey) * TDE_GCAT_KEYS_COUNT); +} + +void +TDEGlCatKeyInit(void) +{ + char db_map_path[MAXPGPATH] = {0}; + + init_keyring(); + + pg_tde_set_db_file_paths(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), + db_map_path, NULL); + if (access(db_map_path, F_OK) == -1) + { + init_gl_catalog_keys(); + } +} + +TDEMasterKey * +TDEGetGlCatKeyFromCache(void) +{ + TDEMasterKey *mkey; + + mkey = &EncryptionState->master_keys[TDE_GCAT_KEY_XLOG]; + if (mkey->keyLength == 0) + return NULL; + + return mkey; +} + +void +TDEPutGlCatKeyInCache(TDEMasterKey *mkey) +{ + memcpy(EncryptionState->master_keys + TDE_GCAT_KEY_XLOG, mkey, sizeof(TDEMasterKey)); +} + +RelKeyData * +GetGlCatInternalKey(Oid obj_id) +{ + return GetInternalKey(GLOBAL_SPACE_RLOCATOR(obj_id), EncryptionState->keyring); +} + +static void +init_keyring(void) +{ + EncryptionState->keyring->type = get_keyring_provider_from_typename(KRingProviderType); + switch (EncryptionState->keyring->type) + { + case FILE_KEY_PROVIDER: + FileKeyring *kring = (FileKeyring *) EncryptionState->keyring; + strncpy(kring->file_name, KRingProviderFilePath, sizeof(kring->file_name)); + break; + } +} + +/* + * Keys are created during the cluster start only, so no locks needed here. + */ +static void +init_gl_catalog_keys(void) +{ + InternalKey int_key; + RelKeyData *rel_key_data; + RelKeyData *enc_rel_key_data; + RelFileLocator *rlocator; + TDEMasterKey *mkey; + + mkey = create_master_key("global-catalog-master-key", + EncryptionState->keyring, + GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID, false); + + memset(&int_key, 0, sizeof(InternalKey)); + + /* Create and store an internal key for XLog */ + if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) + { + ereport(FATAL, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate internal key for \"WAL\": %s", + ERR_error_string(ERR_get_error(), NULL)))); + } + + rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); + rel_key_data = tde_create_rel_key(rlocator->relNumber, &int_key, &mkey->keyInfo); + enc_rel_key_data = tde_encrypt_rel_key(mkey, rel_key_data, rlocator); + pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &mkey->keyInfo); + + TDEPutGlCatKeyInCache(mkey); +} + +static TDEMasterKey * +create_master_key(const char *key_name, GenericKeyring *keyring, + Oid dbOid, Oid spcOid, bool ensure_new_key) +{ + TDEMasterKey *masterKey; + keyInfo *keyInfo = NULL; + + masterKey = palloc(sizeof(TDEMasterKey)); + masterKey->keyInfo.databaseId = dbOid; + masterKey->keyInfo.tablespaceId = spcOid; + masterKey->keyInfo.keyId.version = DEFAULT_MASTER_KEY_VERSION; + masterKey->keyInfo.keyringId = keyring->key_id; + strncpy(masterKey->keyInfo.keyId.name, key_name, TDE_KEY_NAME_LEN); + gettimeofday(&masterKey->keyInfo.creationTime, NULL); + + keyInfo = load_latest_versioned_key_name(&masterKey->keyInfo, keyring, ensure_new_key); + + if (keyInfo == NULL) + keyInfo = KeyringGenerateNewKeyAndStore(keyring, masterKey->keyInfo.keyId.versioned_name, INTERNAL_KEY_LEN, false); + + if (keyInfo == NULL) + { + ereport(ERROR, + (errmsg("failed to retrieve master key"))); + } + + masterKey->keyLength = keyInfo->data.len; + memcpy(masterKey->keyData, keyInfo->data.data, keyInfo->data.len); + + return masterKey; +} \ No newline at end of file diff --git a/src/catalog/tde_master_key.c b/src/catalog/tde_master_key.c index e1b5d913b20..2b97ee0500f 100644 --- a/src/catalog/tde_master_key.c +++ b/src/catalog/tde_master_key.c @@ -29,8 +29,7 @@ #include #include "access/pg_tde_tdemap.h" - -#define DEFAULT_MASTER_KEY_VERSION 1 +#include "catalog/tde_global_catalog.h" typedef struct TdeMasterKeySharedState { @@ -67,7 +66,6 @@ static Size required_shared_mem_size(void); static int required_locks_count(void); static void shared_memory_shutdown(int code, Datum arg); static void master_key_startup_cleanup(int tde_tbl_count, void *arg); -static keyInfo *load_latest_versioned_key_name(TDEMasterKeyInfo *mastere_key_info, GenericKeyring *keyring, bool ensure_new_key); static void clear_master_key_cache(Oid databaseId, Oid tablespaceId) ; static inline dshash_table *get_master_key_Hash(void); static TDEMasterKey *get_master_key_from_cache(Oid dbOid); @@ -250,7 +248,11 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) LWLockAcquire(lock_files, LW_SHARED); LWLockAcquire(lock_cache, LW_EXCLUSIVE); - masterKey = get_master_key_from_cache(dbOid); + /* Global catalog has its own cache */ + if (spcOid == GLOBALTABLESPACE_OID) + masterKey = TDEGetGlCatKeyFromCache(); + else + masterKey = get_master_key_from_cache(dbOid); if (masterKey) { @@ -299,7 +301,10 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) masterKey->keyLength = keyInfo->data.len; Assert(MyDatabaseId == masterKey->keyInfo.databaseId); - push_master_key_to_cache(masterKey); + if (spcOid == GLOBALTABLESPACE_OID) + TDEPutGlCatKeyInCache(masterKey); + else + push_master_key_to_cache(masterKey); /* Release the exclusive locks here */ LWLockRelease(lock_cache); @@ -376,10 +381,10 @@ set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, save_master_key_info(&masterKey->keyInfo); /* XLog the new key*/ - // XLogBeginInsert(); - // XLogRegisterData((char *) &masterKey->keyInfo, sizeof(TDEMasterKeyInfo)); - // XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_MASTER_KEY); - + XLogBeginInsert(); + XLogRegisterData((char *) &masterKey->keyInfo, sizeof(TDEMasterKeyInfo)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_MASTER_KEY); + push_master_key_to_cache(masterKey); } @@ -481,7 +486,7 @@ xl_tde_perform_rotate_key(XLogMasterKeyRotate *xlrec) * If ensure_new_key is true, then we will keep on incrementing the version number * till we get a key name that is not present in the keyring */ -static keyInfo * +keyInfo * load_latest_versioned_key_name(TDEMasterKeyInfo *mastere_key_info, GenericKeyring *keyring, bool ensure_new_key) { KeyringReturnCodes kr_ret; diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index 4b72bf5efb1..471deb64194 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -15,16 +15,6 @@ #include "storage/fd.h" #include "storage/relfilelocator.h" -/* - * Neeed for glogbal data (WAL etc) keys identification in caches and storage. - * We take IDs the oid type operators, so there is no overlap with the "real" - * catalog object possible. - */ -#define GLOBAL_DATA_TDE_OID 607 /* Global objects fake "db" */ -#define XLOG_TDE_OID 608 - -#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) {GLOBALTABLESPACE_OID, 0, _obj_oid} - typedef struct InternalKey { uint8 key[INTERNAL_KEY_LEN]; diff --git a/src/include/access/pg_tde_xlog.h b/src/include/access/pg_tde_xlog.h index c380b8b207f..740eda8de3d 100644 --- a/src/include/access/pg_tde_xlog.h +++ b/src/include/access/pg_tde_xlog.h @@ -37,15 +37,15 @@ static const RmgrData pg_tde_rmgr = { .rm_identify = pg_tde_rmgr_identify }; +<<<<<<< HEAD #ifdef PERCONA_FORK /* XLog encryption staff */ +======= +/* XLog encryption stuff */ +>>>>>>> e9805ad (Create and use global catalog key) -/* GUC */ -extern bool EncryptXLog; - -extern Size TDEXLogEncryptBuffSize(); -extern Size XLogEncStateSize(); +extern Size TDEXLogEncryptBuffSize(void); #define XLOG_TDE_ENC_BUFF_ALIGNED_SIZE add_size(TDEXLogEncryptBuffSize(), PG_IO_ALIGN_SIZE) @@ -59,9 +59,9 @@ static const XLogSmgr tde_xlog_smgr = { .seg_write = pg_tde_xlog_seg_write, }; -extern void TDEInitXLogSmgr(void); +extern void TDEXLogSmgrInit(void); -extern void xlogInitGUC(void); +extern void XLogInitGUC(void); #endif diff --git a/src/include/catalog/tde_global_catalog.h b/src/include/catalog/tde_global_catalog.h new file mode 100644 index 00000000000..1e32e40875a --- /dev/null +++ b/src/include/catalog/tde_global_catalog.h @@ -0,0 +1,37 @@ +/*------------------------------------------------------------------------- + * + * tde_global_catalog.h + * Global catalog key management + * + * src/include/catalog/tde_global_catalog.h + * + *------------------------------------------------------------------------- + */ + +#ifndef TDE_GLOBAL_CATALOG_H +#define TDE_GLOBAL_CATALOG_H + +#include "postgres.h" + +#include "catalog/tde_master_key.h" + +/* + * Needed for glogbal data (WAL etc) keys identification in caches and storage. + * We take IDs the oid type operators, so there is no overlap with the "real" + * catalog object possible. + */ +#define GLOBAL_DATA_TDE_OID 607 /* Global objects fake "db" */ +#define XLOG_TDE_OID 608 + +#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) {GLOBALTABLESPACE_OID, 0, _obj_oid} + +extern void TDEGlCatInitGUC(void); +extern Size TDEGlCatEncStateSize(void); +extern void TDEGlCatShmemInit(void); +extern void TDEGlCatKeyInit(void); + +extern TDEMasterKey *TDEGetGlCatKeyFromCache(void); +extern void TDEPutGlCatKeyInCache(TDEMasterKey *mkey); +extern RelKeyData *GetGlCatInternalKey(Oid obj_id); + +#endif /*TDE_GLOBAL_CATALOG_H*/ diff --git a/src/include/catalog/tde_master_key.h b/src/include/catalog/tde_master_key.h index a6f82123ead..2f70c9c9b9f 100644 --- a/src/include/catalog/tde_master_key.h +++ b/src/include/catalog/tde_master_key.h @@ -17,6 +17,7 @@ #include "nodes/pg_list.h" #include "storage/lwlock.h" +#define DEFAULT_MASTER_KEY_VERSION 1 #define MASTER_KEY_NAME_LEN TDE_KEY_NAME_LEN #define MAX_MASTER_KEY_VERSION_NUM 100000 @@ -76,5 +77,8 @@ extern TDEMasterKey *set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, Oid dbOid, Oid spcOid, bool ensure_new_key); - +extern keyInfo *load_latest_versioned_key_name(TDEMasterKeyInfo *mastere_key_info, + GenericKeyring *keyring, + bool ensure_new_key); + #endif /*PG_TDE_MASTER_KEY_H*/ diff --git a/src/keyring/.keyring_api.c.swp b/src/keyring/.keyring_api.c.swp deleted file mode 100644 index e730fc72a57d6acba563fe1e2dd8fd54b82c05d1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2O>7%Q6vro}e3(+&Qd9|n&@`yXHg?>mEg)K_vSL$(q)z3eEeKI}vpcp|+3c=% z$4%3!fW(Clfy50|;DQi`3JFeJ5EUvQ4u}Js5K#4q5aj@O%KwdbFHiDbL zwatuu1-=9ygV(`p;2BT>C9oG1Kn~mw?gQW4!PwW}Bk&G*8@vFX1P_5M*aS9!>zf$+ z4O|2lz#G5-MX(k8c{^j5zyc$ElnH zEQRa1p?Yj)1}!PS5eSh+cLrDISjukj^mOMa&z$rEf#1|iM|*NKOODG{;4Rlh5O_g` z?=14^%2Tze>f!q2Y`IdOJ2p94nVZWDTq~7I>G7s;MPNullaFRH zWJXE^!uJB1PAtU_@k*^WTjP)M6Qx=e3+7hs#htuZVjXq7<-7&!5CcohU0i zGTjF(^u0@X4V}oLkqeT;{uWtOI8IO5Z_t>Dk z#`@$>O%&k|uzf)9#BQ^+-Oh;~w<4CJ626*QWY@+J|OxT0mdBjEP`AY6xqgx-I* zJoOi&TwEy=LE{N6hwLgDv<2dDL@as1$|Mem;ymcwvMtTlyqESIi_VOqiDK2wc$nKw z&mwmWAJxf94<3Ns;j0pXtWg`NsP+*iTpcWUQ?KoW~l1a zwHIRtyEO0lJso{X*f5-s^jJ?7Fq}Xb)=FPAYEcnP+#4170l|!%$?-UM=Bb=2 zmlIFVs0>hHAmaBOWe>gJs5czS9J-N8W@m!b8v7}w5_X}EW}Jo{gfi`z6m`tiz{A5Z zTO2J^nBsz-ybH0KL7krzj>EH#NuIB*!~+}(r6FyT_GH`%0~j0Fyq*aUFfxNS^dKwY zYjq0mgY10IitsXCeN}6;4UY}EM|xVK!SMpqHf7IzILm6grqi|rKN!7`6avGw##$4J z9z(q&P+PZg6whrZEG&s7^<%6xn&@-s0fS@ebq6l?cG2_QXo^DE!mYhshG~j0ESzz? xW^YG)yq#JTt1Nb&X{!g^QP~=H->v;)P8)5f&!YYz4ZH~VuHQd`PSTSpe*ybIVLbo< diff --git a/src/pg_tde.c b/src/pg_tde.c index 563b60ac523..bac49ea6fcd 100644 --- a/src/pg_tde.c +++ b/src/pg_tde.c @@ -32,6 +32,7 @@ #include "utils/builtins.h" #include "pg_tde_defs.h" #include "smgr/pg_tde_smgr.h" +#include "catalog/tde_global_catalog.h" #define MAX_ON_INSTALLS 5 @@ -80,10 +81,12 @@ tde_shmem_startup(void) TdeShmemInit(); AesInit(); -#ifdef PERCONA_FORK + + TDEGlCatShmemInit(); + TDEGlCatKeyInit(); + TDEXLogShmemInit(); - TDEInitXLogSmgr(); -#endif + TDEXLogSmgrInit(); } void @@ -96,9 +99,8 @@ _PG_init(void) keyringRegisterVariables(); InitializeMasterKeyInfo(); -#ifdef PERCONA_FORK - xlogInitGUC(); -#endif + XLogInitGUC(); + TDEGlCatInitGUC(); prev_shmem_request_hook = shmem_request_hook; shmem_request_hook = tde_shmem_request; From 5b4672a026fb644554df62de5460e30473b2b092 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Wed, 5 Jun 2024 18:40:38 +0300 Subject: [PATCH 09/14] Use Linux fd syscalls instead of PG's File Vfd During the server start, when pg_tde module is loading and it needs to read *.map, *.dat file, InitFileAccess is yet to be called, hence Vfd isn't ready to use. The same gonna happen during recovery. So use raw pread/pwrite calls istead. --- src/access/pg_tde_tdemap.c | 276 +++++++++++++---------- src/access/pg_tde_xlog.c | 16 -- src/catalog/tde_global_catalog.c | 83 ++++--- src/catalog/tde_master_key.c | 11 +- src/include/access/pg_tde_tdemap.h | 2 +- src/include/catalog/tde_global_catalog.h | 6 +- src/keyring/keyring_file.c | 69 +++--- 7 files changed, 261 insertions(+), 202 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 13ac03d7d50..9520e4c777b 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -81,24 +81,24 @@ typedef struct TDEMapFilePath static void put_key_into_map(Oid rel_id, RelKeyData *key); -static File pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignore_missing); -static File pg_tde_file_header_write(char *tde_filename, File tde_file, TDEMasterKeyInfo *master_key_info, off_t *bytes_written); -static File pg_tde_file_header_read(char *tde_filename, File tde_file, TDEFileHeader *fheader, bool *is_new_file, off_t *bytes_read); +static int pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignore_missing); +static int pg_tde_file_header_write(char *tde_filename, int fd, TDEMasterKeyInfo *master_key_info, off_t *bytes_written); +static int pg_tde_file_header_read(char *tde_filename, int fd, TDEFileHeader *fheader, bool *is_new_file, off_t *bytes_read); -static File pg_tde_open_file(char *tde_filename, TDEMasterKeyInfo *master_key_info, bool should_fill_info, int fileFlags, bool *is_new_file, off_t *offset); +static int pg_tde_open_file(char *tde_filename, TDEMasterKeyInfo *master_key_info, bool should_fill_info, int fileFlags, bool *is_new_file, off_t *offset); static int32 pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMasterKeyInfo *master_key_info); -static off_t pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset); +static off_t pg_tde_write_one_map_entry(int fd, const RelFileLocator *rlocator, int flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset); static int32 pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_t *offset, bool should_delete); -static bool pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rlocator, int flags, TDEMapEntry *map_entry, off_t *offset); +static bool pg_tde_read_one_map_entry(int fd, const RelFileLocator *rlocator, int flags, TDEMapEntry *map_entry, off_t *offset); static void pg_tde_write_keydata(char *db_keydata_path, TDEMasterKeyInfo *master_key_info, int32 key_index, RelKeyData *enc_rel_key_data); -static void pg_tde_write_one_keydata(File keydata_file, int32 key_index, RelKeyData *enc_rel_key_data); +static void pg_tde_write_one_keydata(int keydata_fd, int32 key_index, RelKeyData *enc_rel_key_data); static RelKeyData* pg_tde_get_key_from_file(const RelFileLocator *rlocator, GenericKeyring *keyring); static RelKeyData* pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEMasterKey *master_key); -static RelKeyData* pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master_key); +static RelKeyData* pg_tde_read_one_keydata(int keydata_fd, int32 key_index, TDEMasterKey *master_key); -static File keyrotation_init_file(TDEMasterKeyInfo *new_master_key_info, char *rotated_filename, char *filename, bool *is_new_file, off_t *curr_pos); +static int keyrotation_init_file(TDEMasterKeyInfo *new_master_key_info, char *rotated_filename, char *filename, bool *is_new_file, off_t *curr_pos); static void finalize_key_rotation(char *m_path_old, char *k_path_old, char *m_path_new, char *k_path_new); /* @@ -188,7 +188,7 @@ GetRelationKey(RelFileLocator rel) } RelKeyData * -GetInternalKey(RelFileLocator rel, GenericKeyring *keyring) +GetRelationKeyWithKeyring(RelFileLocator rel, GenericKeyring *keyring) { RelKey *curr; RelKeyData *key; @@ -343,8 +343,8 @@ pg_tde_delete_tde_files(Oid dbOid, Oid spcOid) bool pg_tde_save_master_key(TDEMasterKeyInfo *master_key_info) { - File map_file = -1; - File keydata_file = -1; + int map_fd = -1; + int keydata_fd = -1; off_t curr_pos = 0; bool is_new_map = false; bool is_new_key_data = false; @@ -361,12 +361,12 @@ pg_tde_save_master_key(TDEMasterKeyInfo *master_key_info) ereport(LOG, (errmsg("pg_tde_save_master_key"))); /* Create or truncate these map and keydata files. */ - map_file = pg_tde_open_file(db_map_path, master_key_info, false, O_RDWR | O_CREAT | O_TRUNC, &is_new_map, &curr_pos); - keydata_file = pg_tde_open_file(db_keydata_path, master_key_info, false, O_RDWR | O_CREAT | O_TRUNC, &is_new_key_data, &curr_pos); + map_fd = pg_tde_open_file(db_map_path, master_key_info, false, O_RDWR | O_CREAT | O_TRUNC, &is_new_map, &curr_pos); + keydata_fd = pg_tde_open_file(db_keydata_path, master_key_info, false, O_RDWR | O_CREAT | O_TRUNC, &is_new_key_data, &curr_pos); /* Closing files. */ - FileClose(map_file); - FileClose(keydata_file); + close(map_fd); + close(keydata_fd); return (is_new_map && is_new_key_data); } @@ -378,7 +378,7 @@ pg_tde_save_master_key(TDEMasterKeyInfo *master_key_info) TDEMasterKeyInfo * pg_tde_get_master_key(Oid dbOid, Oid spcOid) { - File tde_file = -1; + int fd = -1; TDEFileHeader fheader; TDEMasterKeyInfo *master_key_info = NULL; bool is_new_file = false; @@ -397,15 +397,15 @@ pg_tde_get_master_key(Oid dbOid, Oid spcOid) * Ensuring that we always open the file in binary mode. The caller must * specify other flags for reading, writing or creating the file. */ - tde_file = pg_tde_open_file_basic(db_map_path, O_RDONLY, true); + fd = pg_tde_open_file_basic(db_map_path, O_RDONLY, true); /* The file does not exist. */ - if (tde_file < 0) + if (fd < 0) return NULL; - pg_tde_file_header_read(db_map_path, tde_file, &fheader, &is_new_file, &bytes_read); + pg_tde_file_header_read(db_map_path, fd, &fheader, &is_new_file, &bytes_read); - FileClose(tde_file); + close(fd); /* It's not a new file. So we can memcpy the master key info from the header */ if (!is_new_file) @@ -425,32 +425,32 @@ pg_tde_get_master_key(Oid dbOid, Oid spcOid) * Returns the file descriptor in case of a success. Otherwise, fatal error * is raised except when ignore_missing is true and the file does not exit. */ -static File +static int pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignore_missing) { - File tde_file = -1; + int fd = -1; /* * Ensuring that we always open the file in binary mode. The caller must * specify other flags for reading, writing or creating the file. */ - tde_file = PathNameOpenFile(tde_filename, fileFlags | PG_BINARY); - if (tde_file < 0 && !(errno == ENOENT && ignore_missing == true)) + fd = BasicOpenFile(tde_filename, fileFlags | PG_BINARY); + if (fd < 0 && !(errno == ENOENT && ignore_missing == true)) { ereport(PANIC, (errcode_for_file_access(), - errmsg("Could not open tde file \"%s\": %m", + errmsg("could not open tde file \"%s\": %m", tde_filename))); } - return tde_file; + return fd; } /* * Write TDE file header to a TDE file. */ -static File -pg_tde_file_header_write(char *tde_filename, File tde_file, TDEMasterKeyInfo *master_key_info, off_t *bytes_written) +static int +pg_tde_file_header_write(char *tde_filename, int fd, TDEMasterKeyInfo *master_key_info, off_t *bytes_written) { TDEFileHeader fheader; size_t sz = sizeof(TDEMasterKeyInfo); @@ -464,33 +464,42 @@ pg_tde_file_header_write(char *tde_filename, File tde_file, TDEMasterKeyInfo *ma memset(&fheader.master_key_info, 0, sz); memcpy(&fheader.master_key_info, master_key_info, sz); - *bytes_written = FileWrite(tde_file, &fheader, TDE_FILE_HEADER_SIZE, 0, WAIT_EVENT_DATA_FILE_WRITE); + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + *bytes_written = pg_pwrite(fd, &fheader, TDE_FILE_HEADER_SIZE, 0); if (*bytes_written != TDE_FILE_HEADER_SIZE) { ereport(ERROR, (errcode_for_file_access(), - errmsg("Could not write tde file \"%s\": %m", + errmsg("could not write tde file \"%s\": %m", tde_filename))); } - return tde_file; + if (pg_fsync(fd) != 0) + { + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", tde_filename))); + } + + return fd; } /* * Read TDE file header from a TDE file and fill in the fheader data structure. */ -static File -pg_tde_file_header_read(char *tde_filename, File tde_file, TDEFileHeader *fheader, bool *is_new_file, off_t *bytes_read) +static int +pg_tde_file_header_read(char *tde_filename, int fd, TDEFileHeader *fheader, bool *is_new_file, off_t *bytes_read) { Assert(fheader); - *bytes_read = FileRead(tde_file, fheader, TDE_FILE_HEADER_SIZE, 0, WAIT_EVENT_DATA_FILE_READ); + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + *bytes_read = pg_pread(fd, fheader, TDE_FILE_HEADER_SIZE, 0); *is_new_file = (*bytes_read == 0); /* File doesn't exist */ if (*bytes_read == 0) - return tde_file; + return fd; if (*bytes_read != TDE_FILE_HEADER_SIZE || fheader->file_version != PG_TDE_FILEMAGIC) @@ -502,7 +511,7 @@ pg_tde_file_header_read(char *tde_filename, File tde_file, TDEFileHeader *fheade tde_filename))); } - return tde_file; + return fd; } /* @@ -522,10 +531,10 @@ pg_tde_file_header_read(char *tde_filename, File tde_file, TDEFileHeader *fheade * The caller can pass the required flags to ensure that file is created * or an error is thrown if the file does not exist. */ -File +int pg_tde_open_file(char *tde_filename, TDEMasterKeyInfo *master_key_info, bool should_fill_info, int fileFlags, bool *is_new_file, off_t *curr_pos) { - File tde_file = -1; + int fd = -1; TDEFileHeader fheader; off_t bytes_read = 0; off_t bytes_written = 0; @@ -534,16 +543,16 @@ pg_tde_open_file(char *tde_filename, TDEMasterKeyInfo *master_key_info, bool sho * Ensuring that we always open the file in binary mode. The caller must * specify other flags for reading, writing or creating the file. */ - tde_file = pg_tde_open_file_basic(tde_filename, fileFlags, false); + fd = pg_tde_open_file_basic(tde_filename, fileFlags, false); - pg_tde_file_header_read(tde_filename, tde_file, &fheader, is_new_file, &bytes_read); + pg_tde_file_header_read(tde_filename, fd, &fheader, is_new_file, &bytes_read); /* In case it's a new file, let's add the header now. */ if (*is_new_file && master_key_info) - pg_tde_file_header_write(tde_filename, tde_file, master_key_info, &bytes_written); + pg_tde_file_header_write(tde_filename, fd, master_key_info, &bytes_written); *curr_pos = bytes_read + bytes_written; - return tde_file; + return fd; } /* @@ -558,7 +567,7 @@ pg_tde_open_file(char *tde_filename, TDEMasterKeyInfo *master_key_info, bool sho static int32 pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMasterKeyInfo *master_key_info) { - File map_file = -1; + int map_fd = -1; int32 key_index = 0; TDEMapEntry map_entry; bool is_new_file; @@ -567,7 +576,7 @@ pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMas bool found = false; /* Open and vaidate file for basic correctness. */ - map_file = pg_tde_open_file(db_map_path, master_key_info, false, O_RDWR | O_CREAT, &is_new_file, &curr_pos); + map_fd = pg_tde_open_file(db_map_path, master_key_info, false, O_RDWR | O_CREAT, &is_new_file, &curr_pos); prev_pos = curr_pos; /* @@ -578,7 +587,7 @@ pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMas while(1) { prev_pos = curr_pos; - found = pg_tde_read_one_map_entry(map_file, NULL, MAP_ENTRY_FREE, &map_entry, &curr_pos); + found = pg_tde_read_one_map_entry(map_fd, NULL, MAP_ENTRY_FREE, &map_entry, &curr_pos); /* We either reach EOF or found an empty slot in the middle of the file */ if (prev_pos == curr_pos || found) @@ -590,10 +599,10 @@ pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMas /* Write the given entry at the location pointed by prev_pos; i.e. the free entry */ curr_pos = prev_pos; - pg_tde_write_one_map_entry(map_file, rlocator, MAP_ENTRY_VALID, key_index, &map_entry, &prev_pos); + pg_tde_write_one_map_entry(map_fd, rlocator, MAP_ENTRY_VALID, key_index, &map_entry, &prev_pos); /* Let's close the file. */ - FileClose(map_file); + close(map_fd); /* Register the entry to be freed in case the transaction aborts */ RegisterEntryForDeletion(rlocator, curr_pos, false); @@ -606,7 +615,7 @@ pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMas * map file. */ static off_t -pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset) +pg_tde_write_one_map_entry(int fd, const RelFileLocator *rlocator, int flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset) { int bytes_written = 0; @@ -617,7 +626,8 @@ pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int fl map_entry->flags = flags; map_entry->key_index = key_index; - bytes_written = FileWrite(map_file, map_entry, MAP_ENTRY_SIZE, *offset, WAIT_EVENT_DATA_FILE_WRITE); + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + bytes_written = pg_pwrite(fd, map_entry, MAP_ENTRY_SIZE, *offset); /* Add the entry to the file */ if (bytes_written != MAP_ENTRY_SIZE) @@ -626,9 +636,17 @@ pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int fl pg_tde_set_db_file_paths(rlocator, db_map_path, NULL); ereport(FATAL, (errcode_for_file_access(), - errmsg("Could not write tde map file \"%s\": %m", + errmsg("could not write tde map file \"%s\": %m", db_map_path))); } + if (pg_fsync(fd) != 0) + { + char db_map_path[MAXPGPATH] = {0}; + pg_tde_set_db_file_paths(rlocator, db_map_path, NULL); + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", db_map_path))); + } return (*offset + bytes_written); } @@ -645,7 +663,7 @@ pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int fl static int32 pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_t *offset, bool should_delete) { - File map_file = -1; + File map_fd = -1; int32 key_index = 0; TDEMapEntry map_entry; bool is_new_file; @@ -659,7 +677,7 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ * Open and validate file for basic correctness. DO NOT create it. * The file should pre-exist otherwise we should never be here. */ - map_file = pg_tde_open_file(db_map_path, NULL, false, O_RDWR, &is_new_file, &curr_pos); + map_fd = pg_tde_open_file(db_map_path, NULL, false, O_RDWR, &is_new_file, &curr_pos); /* * If we need to delete an entry, we expect an offset value to the start @@ -668,13 +686,13 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ */ if (should_delete == true && *offset > 0) { - curr_pos = lseek(map_file, *offset, SEEK_SET); + curr_pos = lseek(map_fd, *offset, SEEK_SET); if (curr_pos == -1) { ereport(FATAL, (errcode_for_file_access(), - errmsg("Could not seek in tde map file \"%s\": %m", + errmsg("could not seek in tde map file \"%s\": %m", db_map_path))); } } @@ -692,7 +710,7 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ while(1) { prev_pos = curr_pos; - found = pg_tde_read_one_map_entry(map_file, rlocator, MAP_ENTRY_VALID, &map_entry, &curr_pos); + found = pg_tde_read_one_map_entry(map_fd, rlocator, MAP_ENTRY_VALID, &map_entry, &curr_pos); /* We've reached EOF */ if (curr_pos == prev_pos) @@ -704,7 +722,7 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ /* Mark the entry pointed by prev_pos as free */ if (should_delete) { - pg_tde_write_one_map_entry(map_file, NULL, MAP_ENTRY_FREE, 0, &map_entry, &prev_pos); + pg_tde_write_one_map_entry(map_fd, NULL, MAP_ENTRY_FREE, 0, &map_entry, &prev_pos); } break; @@ -715,7 +733,7 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ } /* Let's close the file. */ - FileClose(map_file); + close(map_fd); /* Return -1 indicating that no entry was removed */ return ((found) ? key_index : -1); @@ -742,7 +760,8 @@ pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rlocator, int fla Assert(offset); /* Read the entry at the given offset */ - bytes_read = FileRead(map_file, map_entry, MAP_ENTRY_SIZE, *offset, WAIT_EVENT_DATA_FILE_READ); + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + bytes_read = pg_pread(map_file, map_entry, MAP_ENTRY_SIZE, *offset); /* We've reached the end of the file. */ if (bytes_read != MAP_ENTRY_SIZE) @@ -771,38 +790,46 @@ pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rlocator, int fla static void pg_tde_write_keydata(char *db_keydata_path, TDEMasterKeyInfo *master_key_info, int32 key_index, RelKeyData *enc_rel_key_data) { - File keydata_file = -1; + File fd = -1; bool is_new_file; off_t curr_pos = 0; /* Open and validate file for basic correctness. */ - keydata_file = pg_tde_open_file(db_keydata_path, master_key_info, false, O_RDWR | O_CREAT, &is_new_file, &curr_pos); + fd = pg_tde_open_file(db_keydata_path, master_key_info, false, O_RDWR | O_CREAT, &is_new_file, &curr_pos); /* Write a single key data */ - pg_tde_write_one_keydata(keydata_file, key_index, enc_rel_key_data); + pg_tde_write_one_keydata(fd, key_index, enc_rel_key_data); /* Let's close the file. */ - FileClose(keydata_file); + close(fd); } /* * Function writes a single RelKeyData into the file at the given index. */ static void -pg_tde_write_one_keydata(File keydata_file, int32 key_index, RelKeyData *enc_rel_key_data) +pg_tde_write_one_keydata(int fd, int32 key_index, RelKeyData *enc_rel_key_data) { off_t curr_pos; - Assert(keydata_file != -1); + Assert(fd != -1); /* Calculate the writing position in the file. */ curr_pos = (key_index * INTERNAL_KEY_LEN) + TDE_FILE_HEADER_SIZE; - if (FileWrite(keydata_file, &enc_rel_key_data->internal_key, INTERNAL_KEY_LEN, curr_pos, WAIT_EVENT_DATA_FILE_WRITE) != INTERNAL_KEY_LEN) + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + if (pg_pwrite(fd, &enc_rel_key_data->internal_key, INTERNAL_KEY_LEN, curr_pos) != INTERNAL_KEY_LEN) { ereport(FATAL, (errcode_for_file_access(), - errmsg("Could not write tde key data file: %m"))); + errmsg("could not write tde key data file: %m"))); + } + + if (pg_fsync(fd) != 0) + { + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file: %m"))); } } @@ -812,7 +839,7 @@ pg_tde_write_one_keydata(File keydata_file, int32 key_index, RelKeyData *enc_rel static RelKeyData * pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEMasterKey *master_key) { - File keydata_file = -1; + int fd = -1; RelKeyData *enc_rel_key_data; off_t read_pos = 0; bool is_new_file; @@ -820,13 +847,13 @@ pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEMasterKey *master /* Open and validate file for basic correctness. */ LWLockAcquire(lock_files, LW_SHARED); - keydata_file = pg_tde_open_file(db_keydata_path, &master_key->keyInfo, false, O_RDONLY, &is_new_file, &read_pos); + fd = pg_tde_open_file(db_keydata_path, &master_key->keyInfo, false, O_RDONLY, &is_new_file, &read_pos); /* Read the encrypted key from file */ - enc_rel_key_data = pg_tde_read_one_keydata(keydata_file, key_index, master_key); + enc_rel_key_data = pg_tde_read_one_keydata(fd, key_index, master_key); /* Let's close the file. */ - FileClose(keydata_file); + close(fd); LWLockRelease(lock_files); return enc_rel_key_data; @@ -836,7 +863,7 @@ pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEMasterKey *master * Reads a single keydata from the file. */ static RelKeyData * -pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master_key) +pg_tde_read_one_keydata(int keydata_fd, int32 key_index, TDEMasterKey *master_key) { RelKeyData *enc_rel_key_data; off_t read_pos = 0; @@ -850,7 +877,7 @@ pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master read_pos += (key_index * INTERNAL_KEY_LEN) + TDE_FILE_HEADER_SIZE; /* Check if the file has a valid key */ - if ((read_pos + INTERNAL_KEY_LEN) > FileSize(keydata_file)) + if ((read_pos + INTERNAL_KEY_LEN) > lseek(keydata_fd, 0, SEEK_END)) { char db_keydata_path[MAXPGPATH] = {0}; pg_tde_set_db_file_paths(&(RelFileLocator) { @@ -860,13 +887,14 @@ pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master NULL, db_keydata_path); ereport(FATAL, (errcode(ERRCODE_NO_DATA_FOUND), - errmsg("Could not find the required key at index %d in tde data file \"%s\": %m", + errmsg("could not find the required key at index %d in tde data file \"%s\": %m", key_index, db_keydata_path))); } /* Read the encrypted key */ - if (FileRead(keydata_file, &(enc_rel_key_data->internal_key), INTERNAL_KEY_LEN, read_pos, WAIT_EVENT_DATA_FILE_READ) != INTERNAL_KEY_LEN) + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + if (pg_pread(keydata_fd, &(enc_rel_key_data->internal_key), INTERNAL_KEY_LEN, read_pos) != INTERNAL_KEY_LEN) { char db_keydata_path[MAXPGPATH] = {0}; pg_tde_set_db_file_paths(&(RelFileLocator) { @@ -876,7 +904,7 @@ pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master NULL, db_keydata_path); ereport(FATAL, (errcode_for_file_access(), - errmsg("Could not read key at index %d in tde key data file \"%s\": %m", + errmsg("could not read key at index %d in tde key data file \"%s\": %m", key_index, db_keydata_path))); } @@ -940,7 +968,7 @@ pg_tde_delete_key_map_entry(const RelFileLocator *rlocator) { ereport(WARNING, (errcode(ERRCODE_NO_DATA_FOUND), - errmsg("Could not find the required map entry for deletion of relation %d in tde map file \"%s\": %m", + errmsg("could not find the required map entry for deletion of relation %d in tde map file \"%s\": %m", rlocator->relNumber, db_map_path))); @@ -984,7 +1012,7 @@ pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset) { ereport(WARNING, (errcode(ERRCODE_NO_DATA_FOUND), - errmsg("Could not find the required map entry for deletion of relation %d in tde map file \"%s\": %m", + errmsg("could not find the required map entry for deletion of relation %d in tde map file \"%s\": %m", rlocator->relNumber, db_map_path))); @@ -1088,8 +1116,8 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key int32 key_index[MASTER_KEY_COUNT] = {0}; RelKeyData *rel_key_data[MASTER_KEY_COUNT]; RelKeyData *enc_rel_key_data[MASTER_KEY_COUNT]; - File m_file[MASTER_KEY_COUNT] = {-1}; - File k_file[MASTER_KEY_COUNT] = {-1}; + int m_fd[MASTER_KEY_COUNT] = {-1}; + int k_fd[MASTER_KEY_COUNT] = {-1}; char m_path[MASTER_KEY_COUNT][MAXPGPATH]; char k_path[MASTER_KEY_COUNT][MAXPGPATH]; TDEMapEntry map_entry; @@ -1121,17 +1149,17 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key LWLockAcquire(lock_cache, LW_EXCLUSIVE); /* Open both files in read only mode. We don't need to track the current position of the keydata file. We always use the key index */ - m_file[OLD_MASTER_KEY] = pg_tde_open_file(m_path[OLD_MASTER_KEY], &master_key->keyInfo, false, O_RDONLY, &is_new_file, &curr_pos[OLD_MASTER_KEY]); - k_file[OLD_MASTER_KEY] = pg_tde_open_file(k_path[OLD_MASTER_KEY], &master_key->keyInfo, false, O_RDONLY, &is_new_file, &read_pos_tmp); + m_fd[OLD_MASTER_KEY] = pg_tde_open_file(m_path[OLD_MASTER_KEY], &master_key->keyInfo, false, O_RDONLY, &is_new_file, &curr_pos[OLD_MASTER_KEY]); + k_fd[OLD_MASTER_KEY] = pg_tde_open_file(k_path[OLD_MASTER_KEY], &master_key->keyInfo, false, O_RDONLY, &is_new_file, &read_pos_tmp); - m_file[NEW_MASTER_KEY] = keyrotation_init_file(&new_master_key->keyInfo, m_path[NEW_MASTER_KEY], m_path[OLD_MASTER_KEY], &is_new_file, &curr_pos[NEW_MASTER_KEY]); - k_file[NEW_MASTER_KEY] = keyrotation_init_file(&new_master_key->keyInfo, k_path[NEW_MASTER_KEY], k_path[OLD_MASTER_KEY], &is_new_file, &read_pos_tmp); + m_fd[NEW_MASTER_KEY] = keyrotation_init_file(&new_master_key->keyInfo, m_path[NEW_MASTER_KEY], m_path[OLD_MASTER_KEY], &is_new_file, &curr_pos[NEW_MASTER_KEY]); + k_fd[NEW_MASTER_KEY] = keyrotation_init_file(&new_master_key->keyInfo, k_path[NEW_MASTER_KEY], k_path[OLD_MASTER_KEY], &is_new_file, &read_pos_tmp); /* Read all entries until EOF */ for(key_index[OLD_MASTER_KEY] = 0; ; key_index[OLD_MASTER_KEY]++) { prev_pos[OLD_MASTER_KEY] = curr_pos[OLD_MASTER_KEY]; - found = pg_tde_read_one_map_entry(m_file[OLD_MASTER_KEY], NULL, MAP_ENTRY_VALID, &map_entry, &curr_pos[OLD_MASTER_KEY]); + found = pg_tde_read_one_map_entry(m_fd[OLD_MASTER_KEY], NULL, MAP_ENTRY_VALID, &map_entry, &curr_pos[OLD_MASTER_KEY]); /* We either reach EOF */ if (prev_pos[OLD_MASTER_KEY] == curr_pos[OLD_MASTER_KEY]) @@ -1147,7 +1175,7 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key rloc.spcOid = DEFAULTTABLESPACE_OID; /* Let's get the decrypted key and re-encrypt it with the new key. */ - enc_rel_key_data[OLD_MASTER_KEY] = pg_tde_read_one_keydata(k_file[OLD_MASTER_KEY], key_index[OLD_MASTER_KEY], master_key); + enc_rel_key_data[OLD_MASTER_KEY] = pg_tde_read_one_keydata(k_fd[OLD_MASTER_KEY], key_index[OLD_MASTER_KEY], master_key); /* Decrypt and re-encrypt keys */ rel_key_data[OLD_MASTER_KEY] = tde_decrypt_rel_key(master_key, enc_rel_key_data[OLD_MASTER_KEY], &rloc); @@ -1155,20 +1183,20 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key /* Write the given entry at the location pointed by prev_pos */ prev_pos[NEW_MASTER_KEY] = curr_pos[NEW_MASTER_KEY]; - curr_pos[NEW_MASTER_KEY] = pg_tde_write_one_map_entry(m_file[NEW_MASTER_KEY], &rloc, MAP_ENTRY_VALID, key_index[NEW_MASTER_KEY], &map_entry, &prev_pos[NEW_MASTER_KEY]); - pg_tde_write_one_keydata(k_file[NEW_MASTER_KEY], key_index[NEW_MASTER_KEY], enc_rel_key_data[NEW_MASTER_KEY]); + curr_pos[NEW_MASTER_KEY] = pg_tde_write_one_map_entry(m_fd[NEW_MASTER_KEY], &rloc, MAP_ENTRY_VALID, key_index[NEW_MASTER_KEY], &map_entry, &prev_pos[NEW_MASTER_KEY]); + pg_tde_write_one_keydata(k_fd[NEW_MASTER_KEY], key_index[NEW_MASTER_KEY], enc_rel_key_data[NEW_MASTER_KEY]); /* Increment the key index for the new master key */ key_index[NEW_MASTER_KEY]++; } /* Close unrotated files */ - FileClose(m_file[OLD_MASTER_KEY]); - FileClose(k_file[OLD_MASTER_KEY]); + close(m_fd[OLD_MASTER_KEY]); + close(k_fd[OLD_MASTER_KEY]); /* Let's calculate sizes */ - map_size = FileSize(m_file[NEW_MASTER_KEY]); - keydata_size = FileSize(k_file[NEW_MASTER_KEY]); + map_size = lseek(m_fd[NEW_MASTER_KEY], 0, SEEK_END); + keydata_size = lseek(k_fd[NEW_MASTER_KEY], 0, SEEK_END); xlrec_size = map_size + keydata_size + SizeoOfXLogMasterKeyRotate; /* palloc and fill in the structure */ @@ -1178,12 +1206,13 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key xlrec->map_size = map_size; xlrec->keydata_size = keydata_size; - FileRead(m_file[NEW_MASTER_KEY], xlrec->buff, xlrec->map_size, 0, WAIT_EVENT_DATA_FILE_READ); - FileRead(k_file[NEW_MASTER_KEY], &xlrec->buff[xlrec->map_size], xlrec->keydata_size, 0, WAIT_EVENT_DATA_FILE_READ); + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + pg_pread(m_fd[NEW_MASTER_KEY], xlrec->buff, xlrec->map_size, 0); + pg_pread(k_fd[NEW_MASTER_KEY], &xlrec->buff[xlrec->map_size], xlrec->keydata_size, 0); /* Close the files */ - FileClose(m_file[NEW_MASTER_KEY]); - FileClose(k_file[NEW_MASTER_KEY]); + close(m_fd[NEW_MASTER_KEY]); + close(k_fd[NEW_MASTER_KEY]); /* Insert the XLog record */ XLogBeginInsert(); @@ -1217,8 +1246,8 @@ pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_ TDEFileHeader *fheader; char m_path_new[MAXPGPATH]; char k_path_new[MAXPGPATH]; - File m_file_new; - File k_file_new; + int m_fd_new; + int k_fd_new; bool is_new_file; off_t curr_pos = 0; off_t read_pos_tmp = 0; @@ -1226,6 +1255,7 @@ pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_ LWLock *lock_cache = tde_lwlock_mk_cache(); char db_map_path[MAXPGPATH] = {0}; char db_keydata_path[MAXPGPATH] = {0}; + bool is_err = false; /* Let's get the header. Buff should start with the map file header. */ fheader = (TDEFileHeader *) m_file_data; @@ -1241,38 +1271,56 @@ pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_ LWLockAcquire(lock_cache, LW_EXCLUSIVE); /* Initialize the new files and set the names */ - m_file_new = keyrotation_init_file(&fheader->master_key_info, m_path_new, db_map_path, &is_new_file, &curr_pos); - k_file_new = keyrotation_init_file(&fheader->master_key_info, k_path_new, db_keydata_path, &is_new_file, &read_pos_tmp); + m_fd_new = keyrotation_init_file(&fheader->master_key_info, m_path_new, db_map_path, &is_new_file, &curr_pos); + k_fd_new = keyrotation_init_file(&fheader->master_key_info, k_path_new, db_keydata_path, &is_new_file, &read_pos_tmp); - if (FileWrite(m_file_new, m_file_data, map_size, 0, WAIT_EVENT_DATA_FILE_WRITE) != map_size) + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + if (pg_pwrite(m_fd_new, m_file_data, map_size, 0) != map_size) { - LWLockRelease(lock_cache); - LWLockRelease(lock_files); - ereport(WARNING, (errcode_for_file_access(), - errmsg("Could not write tde file \"%s\": %m", + errmsg("could not write tde file \"%s\": %m", m_path_new))); + is_err = true; + goto FINALIZE; } - - if (FileWrite(k_file_new, k_file_data, keydata_size, 0, WAIT_EVENT_DATA_FILE_WRITE) != keydata_size) + if (pg_fsync(m_fd_new) != 0) { - LWLockRelease(lock_cache); - LWLockRelease(lock_files); + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", m_path_new))); + is_err = true; + goto FINALIZE; + } + + if (pg_pwrite(k_fd_new, k_file_data, keydata_size, 0) != keydata_size) + { ereport(WARNING, (errcode_for_file_access(), - errmsg("Could not write tde file \"%s\": %m", + errmsg("could not write tde file \"%s\": %m", k_path_new))); + is_err = true; + goto FINALIZE; + } + if (pg_fsync(k_fd_new) != 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", k_path_new))); + is_err = true; + goto FINALIZE; } - FileClose(m_file_new); - FileClose(k_file_new); +FINALIZE: + close(m_fd_new); + close(k_fd_new); - finalize_key_rotation(db_map_path, db_keydata_path, m_path_new, k_path_new); + if (!is_err) + finalize_key_rotation(db_map_path, db_keydata_path, m_path_new, k_path_new); LWLockRelease(lock_cache); LWLockRelease(lock_files); - return true; + return !is_err; } diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index c3c661b6be1..c8a1b48f631 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -40,22 +40,6 @@ static ssize_t TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, static void SetXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, char* iv_prefix); static int XLOGChooseNumBuffers(void); -typedef enum -{ - TDE_GCAT_KEY_XLOG, - - /* must be last */ - TDE_GCAT_KEYS_COUNT -} GlobalCatalogKeyTypes; - -/* TODO: move TDEXLogEncryptBuf here*/ -typedef struct EncryptionStateData -{ - GenericKeyring *keyring; - /* TODO: locking */ - TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT]; - -} EncryptionStateData; /* * TDE fork XLog diff --git a/src/catalog/tde_global_catalog.c b/src/catalog/tde_global_catalog.c index a6c694181fc..429c065ecbb 100644 --- a/src/catalog/tde_global_catalog.c +++ b/src/catalog/tde_global_catalog.c @@ -24,21 +24,24 @@ #include #include +#define MASTER_KEY_DEFAULT_NAME "tde-global-catalog-key" + +/* TODO: not sure if we need an option of multiple master keys for the global catalog */ typedef enum { - TDE_GCAT_KEY_XLOG, + TDE_GCAT_XLOG_KEY, /* must be last */ TDE_GCAT_KEYS_COUNT -} GlobalCatalogKeyTypes; +} GlobalCatalogKeyTypes; typedef struct EncryptionStateData { GenericKeyring *keyring; - TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT]; -} EncryptionStateData; + TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT]; +} EncryptionStateData; -static EncryptionStateData *EncryptionState = NULL; +static EncryptionStateData * EncryptionState = NULL; /* GUC */ static char *KRingProviderType = NULL; @@ -46,9 +49,9 @@ static char *KRingProviderFilePath = NULL; static void init_gl_catalog_keys(void); static void init_keyring(void); -static TDEMasterKey *create_master_key(const char *key_name, - GenericKeyring *keyring, Oid dbOid, Oid spcOid, - bool ensure_new_key); +static TDEMasterKey * create_master_key(const char *key_name, + GenericKeyring * keyring, Oid dbOid, Oid spcOid, + bool ensure_new_key); void TDEGlCatInitGUC(void) @@ -81,7 +84,7 @@ TDEGlCatInitGUC(void) Size TDEGlCatEncStateSize(void) { - Size size; + Size size; size = sizeof(EncryptionStateData); size = add_size(size, sizeof(KeyringProviders)); @@ -92,12 +95,12 @@ TDEGlCatEncStateSize(void) void TDEGlCatShmemInit(void) { - bool foundBuf; - char *allocptr; + bool foundBuf; + char *allocptr; EncryptionState = (EncryptionStateData *) - ShmemInitStruct("TDE XLog Encryption State", - TDEGlCatEncStateSize(), &foundBuf); + ShmemInitStruct("TDE XLog Encryption State", + TDEGlCatEncStateSize(), &foundBuf); allocptr = ((char *) EncryptionState) + MAXALIGN(sizeof(EncryptionStateData)); EncryptionState->keyring = (GenericKeyring *) allocptr; @@ -108,24 +111,29 @@ TDEGlCatShmemInit(void) void TDEGlCatKeyInit(void) { - char db_map_path[MAXPGPATH] = {0}; + char db_map_path[MAXPGPATH] = {0}; init_keyring(); pg_tde_set_db_file_paths(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), - db_map_path, NULL); + db_map_path, NULL); if (access(db_map_path, F_OK) == -1) { init_gl_catalog_keys(); } + else + { + /* put an internal key into the cache */ + GetGlCatInternalKey(XLOG_TDE_OID); + } } TDEMasterKey * TDEGetGlCatKeyFromCache(void) { TDEMasterKey *mkey; - - mkey = &EncryptionState->master_keys[TDE_GCAT_KEY_XLOG]; + + mkey = &EncryptionState->master_keys[TDE_GCAT_XLOG_KEY]; if (mkey->keyLength == 0) return NULL; @@ -133,17 +141,18 @@ TDEGetGlCatKeyFromCache(void) } void -TDEPutGlCatKeyInCache(TDEMasterKey *mkey) +TDEPutGlCatKeyInCache(TDEMasterKey * mkey) { - memcpy(EncryptionState->master_keys + TDE_GCAT_KEY_XLOG, mkey, sizeof(TDEMasterKey)); + memcpy(EncryptionState->master_keys + TDE_GCAT_XLOG_KEY, mkey, sizeof(TDEMasterKey)); } RelKeyData * GetGlCatInternalKey(Oid obj_id) { - return GetInternalKey(GLOBAL_SPACE_RLOCATOR(obj_id), EncryptionState->keyring); + return GetRelationKeyWithKeyring(GLOBAL_SPACE_RLOCATOR(obj_id), EncryptionState->keyring); } +/* TODO: add Vault */ static void init_keyring(void) { @@ -151,27 +160,27 @@ init_keyring(void) switch (EncryptionState->keyring->type) { case FILE_KEY_PROVIDER: - FileKeyring *kring = (FileKeyring *) EncryptionState->keyring; + FileKeyring * kring = (FileKeyring *) EncryptionState->keyring; strncpy(kring->file_name, KRingProviderFilePath, sizeof(kring->file_name)); break; } } -/* +/* * Keys are created during the cluster start only, so no locks needed here. */ static void init_gl_catalog_keys(void) { - InternalKey int_key; - RelKeyData *rel_key_data; - RelKeyData *enc_rel_key_data; - RelFileLocator *rlocator; - TDEMasterKey *mkey; + InternalKey int_key; + RelKeyData *rel_key_data; + RelKeyData *enc_rel_key_data; + RelFileLocator *rlocator; + TDEMasterKey *mkey; - mkey = create_master_key("global-catalog-master-key", - EncryptionState->keyring, - GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID, false); + mkey = create_master_key(MASTER_KEY_DEFAULT_NAME, + EncryptionState->keyring, + GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID, false); memset(&int_key, 0, sizeof(InternalKey)); @@ -180,8 +189,8 @@ init_gl_catalog_keys(void) { ereport(FATAL, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not generate internal key for \"WAL\": %s", - ERR_error_string(ERR_get_error(), NULL)))); + errmsg("could not generate internal key for \"WAL\": %s", + ERR_error_string(ERR_get_error(), NULL)))); } rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); @@ -193,11 +202,11 @@ init_gl_catalog_keys(void) } static TDEMasterKey * -create_master_key(const char *key_name, GenericKeyring *keyring, - Oid dbOid, Oid spcOid, bool ensure_new_key) +create_master_key(const char *key_name, GenericKeyring * keyring, + Oid dbOid, Oid spcOid, bool ensure_new_key) { - TDEMasterKey *masterKey; - keyInfo *keyInfo = NULL; + TDEMasterKey *masterKey; + keyInfo *keyInfo = NULL; masterKey = palloc(sizeof(TDEMasterKey)); masterKey->keyInfo.databaseId = dbOid; @@ -222,4 +231,4 @@ create_master_key(const char *key_name, GenericKeyring *keyring, memcpy(masterKey->keyData, keyInfo->data.data, keyInfo->data.len); return masterKey; -} \ No newline at end of file +} diff --git a/src/catalog/tde_master_key.c b/src/catalog/tde_master_key.c index 2b97ee0500f..dd4b73602db 100644 --- a/src/catalog/tde_master_key.c +++ b/src/catalog/tde_master_key.c @@ -232,7 +232,11 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) recursion++; LWLockAcquire(lock_cache, LW_SHARED); - masterKey = get_master_key_from_cache(dbOid); + /* Global catalog has its own cache */ + if (spcOid == GLOBALTABLESPACE_OID) + masterKey = TDEGetGlCatKeyFromCache(); + else + masterKey = get_master_key_from_cache(dbOid); LWLockRelease(lock_cache); if (masterKey) @@ -273,7 +277,8 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) return NULL; } - if (keyring == NULL) { + if (keyring == NULL) + { keyring = GetKeyProviderByID(masterKeyInfo->keyringId); if (keyring == NULL) { @@ -300,7 +305,7 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) memcpy(masterKey->keyData, keyInfo->data.data, keyInfo->data.len); masterKey->keyLength = keyInfo->data.len; - Assert(MyDatabaseId == masterKey->keyInfo.databaseId); + Assert(dbOid == masterKey->keyInfo.databaseId); if (spcOid == GLOBALTABLESPACE_OID) TDEPutGlCatKeyInCache(masterKey); else diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index 471deb64194..16e0bb83624 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -53,7 +53,7 @@ extern void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator); extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset); extern RelKeyData *GetRelationKey(RelFileLocator rel); -extern RelKeyData *GetInternalKey(RelFileLocator rel, GenericKeyring *keyring); +extern RelKeyData *GetRelationKeyWithKeyring(RelFileLocator rel, GenericKeyring *keyring); extern void pg_tde_delete_tde_files(Oid dbOid, Oid spcOid); diff --git a/src/include/catalog/tde_global_catalog.h b/src/include/catalog/tde_global_catalog.h index 1e32e40875a..5dd44b9e941 100644 --- a/src/include/catalog/tde_global_catalog.h +++ b/src/include/catalog/tde_global_catalog.h @@ -23,7 +23,11 @@ #define GLOBAL_DATA_TDE_OID 607 /* Global objects fake "db" */ #define XLOG_TDE_OID 608 -#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) {GLOBALTABLESPACE_OID, 0, _obj_oid} +#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) { \ + GLOBALTABLESPACE_OID, \ + GLOBAL_DATA_TDE_OID, \ + _obj_oid \ +} extern void TDEGlCatInitGUC(void); extern Size TDEGlCatEncStateSize(void); diff --git a/src/keyring/keyring_file.c b/src/keyring/keyring_file.c index 812e9fab5ef..f5d8648ddc8 100644 --- a/src/keyring/keyring_file.c +++ b/src/keyring/keyring_file.c @@ -1,11 +1,11 @@ /*------------------------------------------------------------------------- * * keyring_file.c - * Implements the file provider keyring - * routines. + * Implements the file provider keyring + * routines. * * IDENTIFICATION - * contrib/pg_tde/src/keyring/keyring_file.c + * contrib/pg_tde/src/keyring/keyring_file.c * *------------------------------------------------------------------------- */ @@ -40,21 +40,21 @@ static keyInfo* get_key_by_name(GenericKeyring* keyring, const char* key_name, bool throw_error, KeyringReturnCodes *return_code) { keyInfo* key = NULL; - File file = -1; + int fd = -1; FileKeyring* file_keyring = (FileKeyring*)keyring; off_t bytes_read = 0; off_t curr_pos = 0; *return_code = KEYRING_CODE_SUCCESS; - file = PathNameOpenFile(file_keyring->file_name, PG_BINARY); - if (file < 0) + fd = BasicOpenFile(file_keyring->file_name, PG_BINARY); + if (fd < 0) return NULL; key = palloc(sizeof(keyInfo)); while(true) { - bytes_read = FileRead(file, key, sizeof(keyInfo), curr_pos, WAIT_EVENT_DATA_FILE_READ); + bytes_read = pg_pread(fd, key, sizeof(keyInfo), curr_pos); curr_pos += bytes_read; if (bytes_read == 0 ) @@ -62,13 +62,13 @@ get_key_by_name(GenericKeyring* keyring, const char* key_name, bool throw_error, /* * Empty keyring file is considered as a valid keyring file that has no keys */ - FileClose(file); + close(fd); pfree(key); return NULL; } if (bytes_read != sizeof(keyInfo)) { - FileClose(file); + close(fd); pfree(key); /* Corrupt file */ *return_code = KEYRING_CODE_DATA_CORRUPTED; @@ -81,21 +81,21 @@ get_key_by_name(GenericKeyring* keyring, const char* key_name, bool throw_error, } if (strncasecmp(key->name.name, key_name, sizeof(key->name.name)) == 0) { - FileClose(file); + close(fd); return key; } } - FileClose(file); + close(fd); pfree(key); - return NULL; + return NULL; } static KeyringReturnCodes set_key_by_name(GenericKeyring* keyring, keyInfo *key, bool throw_error) { - off_t bytes_written = 0; + off_t bytes_written = 0; off_t curr_pos = 0; - File file; + int fd; FileKeyring* file_keyring = (FileKeyring*)keyring; keyInfo *existing_key; KeyringReturnCodes return_code = KEYRING_CODE_SUCCESS; @@ -111,26 +111,35 @@ set_key_by_name(GenericKeyring* keyring, keyInfo *key, bool throw_error) return KEYRING_CODE_INVALID_OPERATION; } - file = PathNameOpenFile(file_keyring->file_name, O_CREAT | O_RDWR | PG_BINARY); - if (file < 0) - { + fd = BasicOpenFile(file_keyring->file_name, O_CREAT | O_RDWR | PG_BINARY); + if (fd < 0) + { ereport(throw_error?ERROR:WARNING, (errcode_for_file_access(), errmsg("Failed to open keyring file %s :%m", file_keyring->file_name))); - return KEYRING_CODE_RESOURCE_NOT_ACCESSABLE; - } + return KEYRING_CODE_RESOURCE_NOT_ACCESSABLE; + } /* Write key to the end of file */ - curr_pos = FileSize(file); - bytes_written = FileWrite(file, key, sizeof(keyInfo), curr_pos, WAIT_EVENT_DATA_FILE_WRITE); + curr_pos = lseek(fd, 0, SEEK_END); + bytes_written = pg_pwrite(fd, key, sizeof(keyInfo), curr_pos); if (bytes_written != sizeof(keyInfo)) - { - FileClose(file); - ereport(throw_error?ERROR:WARNING, - (errcode_for_file_access(), - errmsg("keyring file \"%s\" can't be written: %m", - file_keyring->file_name))); - return KEYRING_CODE_RESOURCE_NOT_ACCESSABLE; - } - FileClose(file); + { + close(fd); + ereport(throw_error?ERROR:WARNING, + (errcode_for_file_access(), + errmsg("keyring file \"%s\" can't be written: %m", + file_keyring->file_name))); + return KEYRING_CODE_RESOURCE_NOT_ACCESSABLE; + } + if (pg_fsync(fd) != 0) + { + close(fd); + ereport(throw_error?ERROR:WARNING, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", + file_keyring->file_name))); + return KEYRING_CODE_RESOURCE_NOT_ACCESSABLE; + } + close(fd); return KEYRING_CODE_SUCCESS; } From b90bb3a02c48b128c46b2c0a1427a2ba99c31141 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Fri, 14 Jun 2024 17:45:29 +0300 Subject: [PATCH 10/14] Fixes and chores after the rebase with smgr --- src/access/pg_tde_tdemap.c | 40 ++++++++---------------------- src/catalog/tde_global_catalog.c | 1 + src/include/access/pg_tde_tdemap.h | 2 ++ src/smgr/pg_tde_smgr.c | 2 +- 4 files changed, 14 insertions(+), 31 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 9520e4c777b..fcb1bd8e24e 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -79,8 +79,6 @@ typedef struct TDEMapFilePath char keydata_path[MAXPGPATH]; } TDEMapFilePath; -static void put_key_into_map(Oid rel_id, RelKeyData *key); - static int pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignore_missing); static int pg_tde_file_header_write(char *tde_filename, int fd, TDEMasterKeyInfo *master_key_info, off_t *bytes_written); static int pg_tde_file_header_read(char *tde_filename, int fd, TDEFileHeader *fheader, bool *is_new_file, off_t *bytes_read); @@ -113,8 +111,7 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator) TDEMasterKey *master_key; XLogRelKey xlrec; - pg_tde_set_db_file_paths(newrlocator->dbOid); - master_key = GetMasterKey(newrlocator->dbOid); + master_key = GetMasterKey(newrlocator->dbOid, newrlocator->spcOid, NULL); if (master_key == NULL) { ereport(ERROR, @@ -168,23 +165,7 @@ RelKey *tde_rel_key_map = NULL; RelKeyData * GetRelationKey(RelFileLocator rel) { - RelKey *curr; - RelKeyData *key; - - Oid rel_id = rel.relNumber; - for (curr = tde_rel_key_map; curr != NULL; curr = curr->next) - { - if (curr->rel_id == rel_id) - { - return curr->key; - } - } - - key = pg_tde_get_key_from_file(&rel, NULL); - - put_key_into_map(rel.relNumber, key); - - return key; + return GetRelationKeyWithKeyring(rel, NULL); } RelKeyData * @@ -206,14 +187,14 @@ GetRelationKeyWithKeyring(RelFileLocator rel, GenericKeyring *keyring) if (key != NULL) { - put_key_into_map(rel.relNumber, key); + pg_tde_put_key_into_map(rel.relNumber, key); } return key; } -static void -put_key_into_map(Oid rel_id, RelKeyData *key) { +void +pg_tde_put_key_into_map(Oid rel_id, RelKeyData *key) { RelKey *new; RelKey *prev = NULL; @@ -256,7 +237,7 @@ tde_create_rel_key(Oid rel_id, InternalKey *key, TDEMasterKeyInfo *master_key_in rel_key_data->internal_key.ctx = NULL; /* Add to the decrypted key to cache */ - put_key_into_map(rel_id, rel_key_data); + pg_tde_put_key_into_map(rel_id, rel_key_data); return rel_key_data; } @@ -294,11 +275,11 @@ pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *map_path, char *k { char *db_path; - /* We use dbOid for the global space for key caches but for the backend - * it should be 0. + /* If this is a global space, than the call might be in a critial section + * (during XLog write) so we can't do GetDatabasePath as it calls palloc() */ if (rlocator->spcOid == GLOBALTABLESPACE_OID) - db_path = GetDatabasePath(0, rlocator->spcOid); + db_path = "global"; else db_path = GetDatabasePath(rlocator->dbOid, rlocator->spcOid); @@ -384,14 +365,13 @@ pg_tde_get_master_key(Oid dbOid, Oid spcOid) bool is_new_file = false; off_t bytes_read = 0; char db_map_path[MAXPGPATH] = {0}; - char db_keydata_path[MAXPGPATH] = {0}; /* Set the file paths */ pg_tde_set_db_file_paths(&(RelFileLocator) { spcOid, dbOid, 0}, - db_map_path, db_keydata_path); + db_map_path, NULL); /* * Ensuring that we always open the file in binary mode. The caller must diff --git a/src/catalog/tde_global_catalog.c b/src/catalog/tde_global_catalog.c index 429c065ecbb..662974a3508 100644 --- a/src/catalog/tde_global_catalog.c +++ b/src/catalog/tde_global_catalog.c @@ -198,6 +198,7 @@ init_gl_catalog_keys(void) enc_rel_key_data = tde_encrypt_rel_key(mkey, rel_key_data, rlocator); pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &mkey->keyInfo); + pg_tde_put_key_into_map(rlocator->relNumber, rel_key_data); TDEPutGlCatKeyInCache(mkey); } diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index 16e0bb83624..e0e06b63753 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -69,4 +69,6 @@ extern void pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *map_p const char * tde_sprint_key(InternalKey *k); +extern void pg_tde_put_key_into_map(Oid rel_id, RelKeyData *key); + #endif /*PG_TDE_MAP_H*/ diff --git a/src/smgr/pg_tde_smgr.c b/src/smgr/pg_tde_smgr.c index c6c2400eade..7c4aa49f7fc 100644 --- a/src/smgr/pg_tde_smgr.c +++ b/src/smgr/pg_tde_smgr.c @@ -35,7 +35,7 @@ tde_smgr_get_key(SMgrRelation reln) recursion++; - if(GetMasterKey(reln->smgr_rlocator.locator.relNumber)==NULL) + if(GetMasterKey(reln->smgr_rlocator.locator.relNumber, reln->smgr_rlocator.locator.spcOid, NULL)==NULL) { recursion--; return NULL; From 33e76da18826438e08b7ecc9a27fed5f90361200 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Mon, 17 Jun 2024 18:32:31 +0300 Subject: [PATCH 11/14] Add todo for key management --- src/catalog/tde_global_catalog.c | 7 ++++++- src/include/access/pg_tde_xlog.h | 4 ---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/catalog/tde_global_catalog.c b/src/catalog/tde_global_catalog.c index 662974a3508..844d8c19c24 100644 --- a/src/catalog/tde_global_catalog.c +++ b/src/catalog/tde_global_catalog.c @@ -152,7 +152,12 @@ GetGlCatInternalKey(Oid obj_id) return GetRelationKeyWithKeyring(GLOBAL_SPACE_RLOCATOR(obj_id), EncryptionState->keyring); } -/* TODO: add Vault */ +/* + * TODO: should be aligned with the rest of the keyring_provider code after its + * refactoring + * + * TODO: add Vault + */ static void init_keyring(void) { diff --git a/src/include/access/pg_tde_xlog.h b/src/include/access/pg_tde_xlog.h index 740eda8de3d..17a1c65ca3a 100644 --- a/src/include/access/pg_tde_xlog.h +++ b/src/include/access/pg_tde_xlog.h @@ -37,13 +37,9 @@ static const RmgrData pg_tde_rmgr = { .rm_identify = pg_tde_rmgr_identify }; -<<<<<<< HEAD #ifdef PERCONA_FORK /* XLog encryption staff */ -======= -/* XLog encryption stuff */ ->>>>>>> e9805ad (Create and use global catalog key) extern Size TDEXLogEncryptBuffSize(void); From 3e007b20acc7008a33d04b9873ac939d313d7964 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Tue, 18 Jun 2024 18:30:59 +0300 Subject: [PATCH 12/14] Fix builds for with upstream PG --- src/access/pg_tde_tdemap.c | 2 +- src/access/pg_tde_xlog.c | 6 +++--- src/catalog/tde_global_catalog.c | 3 +++ src/catalog/tde_master_key.c | 21 +++++++++++++++------ src/pg_tde.c | 7 ++++++- 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index fcb1bd8e24e..71896037924 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -417,7 +417,7 @@ pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignore_missing) fd = BasicOpenFile(tde_filename, fileFlags | PG_BINARY); if (fd < 0 && !(errno == ENOENT && ignore_missing == true)) { - ereport(PANIC, + ereport(ERROR, (errcode_for_file_access(), errmsg("could not open tde file \"%s\": %m", tde_filename))); diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index c8a1b48f631..246ef195c89 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -24,9 +24,9 @@ #include "access/pg_tde_tdemap.h" #include "access/pg_tde_xlog.h" -#include "catalog/tde_global_catalog.h" #include "encryption/enc_tde.h" - +#ifdef PERCONA_FORK +#include "catalog/tde_global_catalog.h" static char *TDEXLogEncryptBuf = NULL; @@ -39,7 +39,7 @@ static XLogPageHeaderData DecryptCurrentPageHrd; static ssize_t TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset); static void SetXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, char* iv_prefix); static int XLOGChooseNumBuffers(void); - +#endif /* * TDE fork XLog diff --git a/src/catalog/tde_global_catalog.c b/src/catalog/tde_global_catalog.c index 844d8c19c24..2c5606997e8 100644 --- a/src/catalog/tde_global_catalog.c +++ b/src/catalog/tde_global_catalog.c @@ -12,6 +12,8 @@ #include "postgres.h" +#ifdef PERCONA_FORK + #include "storage/shmem.h" #include "utils/guc.h" @@ -238,3 +240,4 @@ create_master_key(const char *key_name, GenericKeyring * keyring, return masterKey; } +#endif /* PERCONA_FORK */ diff --git a/src/catalog/tde_master_key.c b/src/catalog/tde_master_key.c index dd4b73602db..4d5070111c3 100644 --- a/src/catalog/tde_master_key.c +++ b/src/catalog/tde_master_key.c @@ -29,7 +29,9 @@ #include #include "access/pg_tde_tdemap.h" +#ifdef PERCONA_FORK #include "catalog/tde_global_catalog.h" +#endif typedef struct TdeMasterKeySharedState { @@ -232,10 +234,12 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) recursion++; LWLockAcquire(lock_cache, LW_SHARED); - /* Global catalog has its own cache */ +#ifdef PERCONA_FORK + /* Global catalog has its own cache */ if (spcOid == GLOBALTABLESPACE_OID) masterKey = TDEGetGlCatKeyFromCache(); - else + else +#endif masterKey = get_master_key_from_cache(dbOid); LWLockRelease(lock_cache); @@ -252,10 +256,12 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) LWLockAcquire(lock_files, LW_SHARED); LWLockAcquire(lock_cache, LW_EXCLUSIVE); +#ifdef PERCONA_FORK /* Global catalog has its own cache */ if (spcOid == GLOBALTABLESPACE_OID) masterKey = TDEGetGlCatKeyFromCache(); - else + else +#endif masterKey = get_master_key_from_cache(dbOid); if (masterKey) @@ -285,8 +291,9 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) LWLockRelease(lock_cache); LWLockRelease(lock_files); - recursion--; - return NULL; + recursion--; + return NULL; + } } keyInfo = KeyringGetKey(keyring, masterKeyInfo->keyId.versioned_name, false, &keyring_ret); @@ -306,9 +313,11 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) masterKey->keyLength = keyInfo->data.len; Assert(dbOid == masterKey->keyInfo.databaseId); +#ifdef PERCONA_FORK if (spcOid == GLOBALTABLESPACE_OID) TDEPutGlCatKeyInCache(masterKey); - else + else +#endif push_master_key_to_cache(masterKey); /* Release the exclusive locks here */ diff --git a/src/pg_tde.c b/src/pg_tde.c index bac49ea6fcd..2298fe67e73 100644 --- a/src/pg_tde.c +++ b/src/pg_tde.c @@ -32,7 +32,9 @@ #include "utils/builtins.h" #include "pg_tde_defs.h" #include "smgr/pg_tde_smgr.h" +#ifdef PERCONA_FORK #include "catalog/tde_global_catalog.h" +#endif #define MAX_ON_INSTALLS 5 @@ -82,11 +84,13 @@ tde_shmem_startup(void) TdeShmemInit(); AesInit(); +#ifdef PERCONA_FORK TDEGlCatShmemInit(); TDEGlCatKeyInit(); TDEXLogShmemInit(); TDEXLogSmgrInit(); +#endif } void @@ -99,9 +103,10 @@ _PG_init(void) keyringRegisterVariables(); InitializeMasterKeyInfo(); +#ifdef PERCONA_FORK XLogInitGUC(); TDEGlCatInitGUC(); - +#endif prev_shmem_request_hook = shmem_request_hook; shmem_request_hook = tde_shmem_request; prev_shmem_startup_hook = shmem_startup_hook; From 9302137c128bb378c1971089f9a7cf4c0cef31f9 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Wed, 19 Jun 2024 17:41:01 +0300 Subject: [PATCH 13/14] Fix key rotation --- src/access/pg_tde_tdemap.c | 1 - src/catalog/tde_master_key.c | 20 +++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 71896037924..5b10f24d8b9 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -1209,7 +1209,6 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key /* Free up the palloc'ed data */ pfree(xlrec); - /* TODO: Remove the existing ones from cache etc. */ return true; #undef OLD_MASTER_KEY diff --git a/src/catalog/tde_master_key.c b/src/catalog/tde_master_key.c index 4d5070111c3..8d1fd7078f1 100644 --- a/src/catalog/tde_master_key.c +++ b/src/catalog/tde_master_key.c @@ -68,7 +68,7 @@ static Size required_shared_mem_size(void); static int required_locks_count(void); static void shared_memory_shutdown(int code, Datum arg); static void master_key_startup_cleanup(int tde_tbl_count, void *arg); -static void clear_master_key_cache(Oid databaseId, Oid tablespaceId) ; +static void clear_master_key_cache(Oid databaseId) ; static inline dshash_table *get_master_key_Hash(void); static TDEMasterKey *get_master_key_from_cache(Oid dbOid); static void push_master_key_to_cache(TDEMasterKey *masterKey); @@ -439,6 +439,7 @@ RotateMasterKey(const char *new_key_name, const char *new_provider_name, bool en TDEMasterKey new_master_key; const keyInfo *keyInfo = NULL; GenericKeyring *keyring; + bool is_rotated; /* * Let's set everything the same as the older master key and @@ -477,8 +478,13 @@ RotateMasterKey(const char *new_key_name, const char *new_provider_name, bool en new_master_key.keyLength = keyInfo->data.len; memcpy(new_master_key.keyData, keyInfo->data.data, keyInfo->data.len); - clear_master_key_cache(MyDatabaseId, MyDatabaseTableSpace); - return pg_tde_perform_rotate_key(master_key, &new_master_key); + is_rotated = pg_tde_perform_rotate_key(master_key, &new_master_key); + if (is_rotated) { + clear_master_key_cache(master_key->keyInfo.databaseId); + push_master_key_to_cache(&new_master_key); + } + + return is_rotated; } /* @@ -490,7 +496,7 @@ xl_tde_perform_rotate_key(XLogMasterKeyRotate *xlrec) bool ret; ret = pg_tde_write_map_keydata_files(xlrec->map_size, xlrec->buff, xlrec->keydata_size, &xlrec->buff[xlrec->map_size]); - clear_master_key_cache(MyDatabaseId, MyDatabaseTableSpace); + clear_master_key_cache(MyDatabaseId); return ret; } @@ -640,7 +646,7 @@ static void push_master_key_to_cache(TDEMasterKey *masterKey) { TDEMasterKey *cacheEntry = NULL; - Oid databaseId = MyDatabaseId; + Oid databaseId = masterKey->keyInfo.databaseId; bool found = false; cacheEntry = dshash_find_or_insert(get_master_key_Hash(), &databaseId, &found); @@ -684,7 +690,7 @@ master_key_startup_cleanup(int tde_tbl_count, void* arg) void cleanup_master_key_info(Oid databaseId, Oid tablespaceId) { - clear_master_key_cache(databaseId, tablespaceId); + clear_master_key_cache(databaseId); /* * TODO: Although should never happen. Still verify if any table in the * database is using tde @@ -695,7 +701,7 @@ cleanup_master_key_info(Oid databaseId, Oid tablespaceId) } static void -clear_master_key_cache(Oid databaseId, Oid tablespaceId) +clear_master_key_cache(Oid databaseId) { TDEMasterKey *cache_entry; From d4181b6b75b2e1fd3f90c452eac33b9db20e7050 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Fri, 21 Jun 2024 16:56:35 +0300 Subject: [PATCH 14/14] Add comment on InternalKey caching --- src/catalog/tde_global_catalog.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/catalog/tde_global_catalog.c b/src/catalog/tde_global_catalog.c index 2c5606997e8..4fb72908582 100644 --- a/src/catalog/tde_global_catalog.c +++ b/src/catalog/tde_global_catalog.c @@ -205,6 +205,11 @@ init_gl_catalog_keys(void) enc_rel_key_data = tde_encrypt_rel_key(mkey, rel_key_data, rlocator); pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &mkey->keyInfo); + /* + * TODO: move global catalog internal keys into own cache. This cache should + * be in the TopMemmoryContext because of SSL contexts + * (see https://github.com/Percona-Lab/pg_tde/pull/214#discussion_r1648998317) + */ pg_tde_put_key_into_map(rlocator->relNumber, rel_key_data); TDEPutGlCatKeyInCache(mkey); }