From 97972f81391bc67606b91fc04848031040f1ed7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20=C3=85strand?= Date: Mon, 25 Aug 2025 13:01:31 +0200 Subject: [PATCH] PG-1870 Enable table encryption by default in TAP This enables table encryption by default in TAP tests when TDE_MODE=1. Use TDE_MODE_SMGR=0 to turn off table encryption when running with pg_tde loaded. The setup for running regress with tde turned on has been slightly modified to match what is done for TAP tests to let tests that run the regress suite under TAP work. --- ci_scripts/tde_setup.sql | 10 ++-- ci_scripts/tde_setup_global.sql | 11 ++--- contrib/amcheck/t/001_verify_heapam.pl | 5 ++ src/bin/pg_amcheck/t/003_check.pl | 5 ++ src/bin/pg_amcheck/t/005_opclass_damage.pl | 5 ++ src/bin/pg_basebackup/t/010_pg_basebackup.pl | 6 +++ src/bin/pg_checksums/t/002_actions.pl | 5 ++ src/bin/pg_dump/t/004_pg_dump_parallel.pl | 6 +++ src/bin/pg_dump/t/010_dump_connstr.pl | 6 +++ src/bin/pg_upgrade/t/002_pg_upgrade.pl | 6 +++ src/bin/pg_upgrade/t/003_logical_slots.pl | 6 +++ src/bin/pg_upgrade/t/004_subscription.pl | 6 +++ src/bin/scripts/t/020_createdb.pl | 6 +++ src/test/perl/PostgreSQL/Test/TdeCluster.pm | 17 +++++++ src/test/recovery/t/014_unlogged_reinit.pl | 5 ++ src/test/recovery/t/016_min_consistency.pl | 5 ++ src/test/recovery/t/018_wal_optimize.pl | 5 ++ src/test/recovery/t/020_archive_status.pl | 6 +++ src/test/recovery/t/032_relfilenode_reuse.pl | 4 ++ src/test/regress/expected/create_am_1.out | 10 ++-- src/test/regress/expected/psql_1.out | 48 ++++++++++---------- 21 files changed, 143 insertions(+), 40 deletions(-) diff --git a/ci_scripts/tde_setup.sql b/ci_scripts/tde_setup.sql index dfce0a1b08c..dd83fba3714 100644 --- a/ci_scripts/tde_setup.sql +++ b/ci_scripts/tde_setup.sql @@ -1,6 +1,6 @@ -CREATE SCHEMA IF NOT EXISTS tde; -CREATE EXTENSION IF NOT EXISTS pg_tde SCHEMA tde; +CREATE SCHEMA IF NOT EXISTS _pg_tde; +CREATE EXTENSION IF NOT EXISTS pg_tde SCHEMA _pg_tde; \! rm -f '/tmp/pg_tde_test_keyring.per' -SELECT tde.pg_tde_add_database_key_provider_file('reg_file-vault', '/tmp/pg_tde_test_keyring.per'); -SELECT tde.pg_tde_create_key_using_database_key_provider('test-db-key', 'reg_file-vault'); -SELECT tde.pg_tde_set_key_using_database_key_provider('test-db-key', 'reg_file-vault'); +SELECT _pg_tde.pg_tde_add_database_key_provider_file('reg_file-vault', '/tmp/pg_tde_test_keyring.per'); +SELECT _pg_tde.pg_tde_create_key_using_database_key_provider('test-db-key', 'reg_file-vault'); +SELECT _pg_tde.pg_tde_set_key_using_database_key_provider('test-db-key', 'reg_file-vault'); diff --git a/ci_scripts/tde_setup_global.sql b/ci_scripts/tde_setup_global.sql index f096285643a..11b440f3e7c 100644 --- a/ci_scripts/tde_setup_global.sql +++ b/ci_scripts/tde_setup_global.sql @@ -1,11 +1,10 @@ -CREATE SCHEMA tde; -CREATE EXTENSION IF NOT EXISTS pg_tde SCHEMA tde; +CREATE SCHEMA IF NOT EXISTS _pg_tde; +CREATE EXTENSION IF NOT EXISTS pg_tde SCHEMA _pg_tde; \! rm -f '/tmp/pg_tde_test_keyring.per' -SELECT tde.pg_tde_add_global_key_provider_file('reg_file-global', '/tmp/pg_tde_test_keyring.per'); -SELECT tde.pg_tde_create_key_using_global_key_provider('server-key', 'reg_file-global'); -SELECT tde.pg_tde_set_server_key_using_global_key_provider('server-key', 'reg_file-global'); +SELECT _pg_tde.pg_tde_add_global_key_provider_file('reg_file-global', '/tmp/pg_tde_test_keyring.per'); +SELECT _pg_tde.pg_tde_create_key_using_global_key_provider('server-key', 'reg_file-global'); +SELECT _pg_tde.pg_tde_set_server_key_using_global_key_provider('server-key', 'reg_file-global'); ALTER SYSTEM SET pg_tde.wal_encrypt = on; ALTER SYSTEM SET default_table_access_method = 'tde_heap'; -ALTER SYSTEM SET search_path = "$user",public,tde; -- restart required diff --git a/contrib/amcheck/t/001_verify_heapam.pl b/contrib/amcheck/t/001_verify_heapam.pl index 9de3148277f..028a75ce06b 100644 --- a/contrib/amcheck/t/001_verify_heapam.pl +++ b/contrib/amcheck/t/001_verify_heapam.pl @@ -9,6 +9,11 @@ use PostgreSQL::Test::Utils; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => "hacks relation files directly for scaffolding"; +} + my ($node, $result); # diff --git a/src/bin/pg_amcheck/t/003_check.pl b/src/bin/pg_amcheck/t/003_check.pl index d99b094dba8..e5156c0b557 100644 --- a/src/bin/pg_amcheck/t/003_check.pl +++ b/src/bin/pg_amcheck/t/003_check.pl @@ -9,6 +9,11 @@ use PostgreSQL::Test::Utils; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => "hacks relation files directly for scaffolding"; +} + my ($node, $port, %corrupt_page, %remove_relation); # Returns the filesystem path for the named relation. diff --git a/src/bin/pg_amcheck/t/005_opclass_damage.pl b/src/bin/pg_amcheck/t/005_opclass_damage.pl index 1eea2152272..a571b79e3ff 100644 --- a/src/bin/pg_amcheck/t/005_opclass_damage.pl +++ b/src/bin/pg_amcheck/t/005_opclass_damage.pl @@ -10,6 +10,11 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => 'investigate why this fails'; +} + my $node = PostgreSQL::Test::Cluster->new('test'); $node->init; $node->start; diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index 54197a256fd..3885e860269 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -16,6 +16,12 @@ if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP}) "pg_basebackup without -E from server with encrypted WAL produces broken backups"; } +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => + 'uses corrupt_page_checksum to directly hack relation files'; +} + program_help_ok('pg_basebackup'); program_version_ok('pg_basebackup'); program_options_handling_ok('pg_basebackup'); diff --git a/src/bin/pg_checksums/t/002_actions.pl b/src/bin/pg_checksums/t/002_actions.pl index 33e7fb53c5e..ec7d7a2e078 100644 --- a/src/bin/pg_checksums/t/002_actions.pl +++ b/src/bin/pg_checksums/t/002_actions.pl @@ -12,6 +12,11 @@ use PostgreSQL::Test::Utils; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => + 'uses corrupt_page_checksum to directly hack relation files'; +} # Utility routine to create and check a table with corrupted checksums # on a wanted tablespace. Note that this stops and starts the node diff --git a/src/bin/pg_dump/t/004_pg_dump_parallel.pl b/src/bin/pg_dump/t/004_pg_dump_parallel.pl index a36ba8cf548..cd9445794bc 100644 --- a/src/bin/pg_dump/t/004_pg_dump_parallel.pl +++ b/src/bin/pg_dump/t/004_pg_dump_parallel.pl @@ -8,6 +8,12 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => + 'pg_restore fail to restore _pg_tde schema on cluster which already has it'; +} + my $dbname1 = 'regression_src'; my $dbname2 = 'regression_dest1'; my $dbname3 = 'regression_dest2'; diff --git a/src/bin/pg_dump/t/010_dump_connstr.pl b/src/bin/pg_dump/t/010_dump_connstr.pl index 54e2e08cc20..df0646ecd3a 100644 --- a/src/bin/pg_dump/t/010_dump_connstr.pl +++ b/src/bin/pg_dump/t/010_dump_connstr.pl @@ -8,6 +8,12 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => + 'pg_restore fail to restore _pg_tde schema on cluster which already has it'; +} + if ($PostgreSQL::Test::Utils::is_msys2) { plan skip_all => 'High bit name tests fail on Msys2'; diff --git a/src/bin/pg_upgrade/t/002_pg_upgrade.pl b/src/bin/pg_upgrade/t/002_pg_upgrade.pl index 78bd776f5be..7ec8538986a 100644 --- a/src/bin/pg_upgrade/t/002_pg_upgrade.pl +++ b/src/bin/pg_upgrade/t/002_pg_upgrade.pl @@ -15,6 +15,12 @@ use PostgreSQL::Test::Utils; use PostgreSQL::Test::AdjustUpgrade; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => + 'pg_restore fail to restore _pg_tde schema on cluster which already has it'; +} + # Can be changed to test the other modes. my $mode = $ENV{PG_TEST_PG_UPGRADE_MODE} || '--copy'; diff --git a/src/bin/pg_upgrade/t/003_logical_slots.pl b/src/bin/pg_upgrade/t/003_logical_slots.pl index 0a2483d3dfc..ecebd366c8d 100644 --- a/src/bin/pg_upgrade/t/003_logical_slots.pl +++ b/src/bin/pg_upgrade/t/003_logical_slots.pl @@ -11,6 +11,12 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => + 'pg_restore fail to restore _pg_tde schema on cluster which already has it'; +} + # Can be changed to test the other modes my $mode = $ENV{PG_TEST_PG_UPGRADE_MODE} || '--copy'; diff --git a/src/bin/pg_upgrade/t/004_subscription.pl b/src/bin/pg_upgrade/t/004_subscription.pl index c59b83af9cc..270a200529f 100644 --- a/src/bin/pg_upgrade/t/004_subscription.pl +++ b/src/bin/pg_upgrade/t/004_subscription.pl @@ -12,6 +12,12 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => + 'pg_restore fail to restore _pg_tde schema on cluster which already has it'; +} + # Can be changed to test the other modes. my $mode = $ENV{PG_TEST_PG_UPGRADE_MODE} || '--copy'; diff --git a/src/bin/scripts/t/020_createdb.pl b/src/bin/scripts/t/020_createdb.pl index 4a0e2c883a1..8fe9c17c6ec 100644 --- a/src/bin/scripts/t/020_createdb.pl +++ b/src/bin/scripts/t/020_createdb.pl @@ -8,6 +8,12 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => + 'tries to use FILE_COPY strategy for database creation with encrypted objects in the template'; +} + program_help_ok('createdb'); program_version_ok('createdb'); program_options_handling_ok('createdb'); diff --git a/src/test/perl/PostgreSQL/Test/TdeCluster.pm b/src/test/perl/PostgreSQL/Test/TdeCluster.pm index c88d9a20bf1..b943bdac08b 100644 --- a/src/test/perl/PostgreSQL/Test/TdeCluster.pm +++ b/src/test/perl/PostgreSQL/Test/TdeCluster.pm @@ -14,6 +14,7 @@ our ($tde_template_dir); BEGIN { $ENV{TDE_MODE_NOSKIP} = 0 unless defined($ENV{TDE_MODE_NOSKIP}); + $ENV{TDE_MODE_SMGR} = 1 unless defined($ENV{TDE_MODE_SMGR}); $ENV{TDE_MODE_WAL} = 1 unless defined($ENV{TDE_MODE_WAL}); } @@ -28,6 +29,22 @@ sub init $self->_tde_init_principal_key; + if ($ENV{TDE_MODE_SMGR}) + { + # Enable the TDE extension in all databases created by initdb, this is + # necessary for the tde_heap access method to be available everywhere. + foreach ('postgres', 'template0', 'template1') + { + _tde_init_sql_command( + $self->data_dir, $_, q( + CREATE SCHEMA _pg_tde; + CREATE EXTENSION pg_tde WITH SCHEMA _pg_tde; + )); + } + $self->SUPER::append_conf('postgresql.conf', + 'default_table_access_method = tde_heap'); + } + if ($ENV{TDE_MODE_WAL}) { $self->SUPER::append_conf('postgresql.conf', diff --git a/src/test/recovery/t/014_unlogged_reinit.pl b/src/test/recovery/t/014_unlogged_reinit.pl index a37fc9249b5..f043bbb5afe 100644 --- a/src/test/recovery/t/014_unlogged_reinit.pl +++ b/src/test/recovery/t/014_unlogged_reinit.pl @@ -12,6 +12,11 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => 'invalid page in block'; +} + my $node = PostgreSQL::Test::Cluster->new('main'); $node->init; diff --git a/src/test/recovery/t/016_min_consistency.pl b/src/test/recovery/t/016_min_consistency.pl index 23d161aebc7..c33892a5a75 100644 --- a/src/test/recovery/t/016_min_consistency.pl +++ b/src/test/recovery/t/016_min_consistency.pl @@ -13,6 +13,11 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => 'reads LSN directly from relation files'; +} + # Find the largest LSN in the set of pages part of the given relation # file. This is used for offline checks of page consistency. The LSN # is historically stored as a set of two numbers of 4 byte-length diff --git a/src/test/recovery/t/018_wal_optimize.pl b/src/test/recovery/t/018_wal_optimize.pl index 3fb31e643b6..e2e29cfa402 100644 --- a/src/test/recovery/t/018_wal_optimize.pl +++ b/src/test/recovery/t/018_wal_optimize.pl @@ -16,6 +16,11 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => 'invalid page in block'; +} + sub check_orphan_relfilenodes { local $Test::Builder::Level = $Test::Builder::Level + 1; diff --git a/src/test/recovery/t/020_archive_status.pl b/src/test/recovery/t/020_archive_status.pl index 9cb178950f8..bdf9176b7ed 100644 --- a/src/test/recovery/t/020_archive_status.pl +++ b/src/test/recovery/t/020_archive_status.pl @@ -10,6 +10,12 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => + q(Failed test 'pg_stat_archiver failed to archive 000000010000000000000004'); +} + my $primary = PostgreSQL::Test::Cluster->new('primary'); $primary->init( has_archiving => 1, diff --git a/src/test/recovery/t/032_relfilenode_reuse.pl b/src/test/recovery/t/032_relfilenode_reuse.pl index 96a8104b802..0c405acabe1 100644 --- a/src/test/recovery/t/032_relfilenode_reuse.pl +++ b/src/test/recovery/t/032_relfilenode_reuse.pl @@ -8,6 +8,10 @@ use PostgreSQL::Test::Utils; use Test::More; use File::Basename; +if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP}) +{ + plan skip_all => 'invalid page in block'; +} my $node_primary = PostgreSQL::Test::Cluster->new('primary'); $node_primary->init(allows_streaming => 1); diff --git a/src/test/regress/expected/create_am_1.out b/src/test/regress/expected/create_am_1.out index f5b9b91674e..822600ef23a 100644 --- a/src/test/regress/expected/create_am_1.out +++ b/src/test/regress/expected/create_am_1.out @@ -129,11 +129,11 @@ ERROR: function int4in(internal) does not exist CREATE ACCESS METHOD bogus TYPE TABLE HANDLER bthandler; ERROR: function bthandler must return type table_am_handler SELECT amname, amhandler, amtype FROM pg_am where amtype = 't' ORDER BY 1, 2; - amname | amhandler | amtype -----------+----------------------+-------- - heap | heap_tableam_handler | t - heap2 | heap_tableam_handler | t - tde_heap | pg_tdeam_handler | t + amname | amhandler | amtype +----------+--------------------------+-------- + heap | heap_tableam_handler | t + heap2 | heap_tableam_handler | t + tde_heap | _pg_tde.pg_tdeam_handler | t (3 rows) -- First create tables employing the new AM using USING diff --git a/src/test/regress/expected/psql_1.out b/src/test/regress/expected/psql_1.out index 5ad2e9d6e4f..80143600713 100644 --- a/src/test/regress/expected/psql_1.out +++ b/src/test/regress/expected/psql_1.out @@ -5013,33 +5013,33 @@ List of access methods \dA: extra argument "bar" ignored \dA+ - List of access methods - Name | Type | Handler | Description -----------+-------+----------------------+---------------------------------------- - brin | Index | brinhandler | block range index (BRIN) access method - btree | Index | bthandler | b-tree index access method - gin | Index | ginhandler | GIN index access method - gist | Index | gisthandler | GiST index access method - hash | Index | hashhandler | hash index access method - heap | Table | heap_tableam_handler | heap table access method - heap2 | Table | heap_tableam_handler | - spgist | Index | spghandler | SP-GiST index access method - tde_heap | Table | pg_tdeam_handler | tde_heap table access method + List of access methods + Name | Type | Handler | Description +----------+-------+--------------------------+---------------------------------------- + brin | Index | brinhandler | block range index (BRIN) access method + btree | Index | bthandler | b-tree index access method + gin | Index | ginhandler | GIN index access method + gist | Index | gisthandler | GiST index access method + hash | Index | hashhandler | hash index access method + heap | Table | heap_tableam_handler | heap table access method + heap2 | Table | heap_tableam_handler | + spgist | Index | spghandler | SP-GiST index access method + tde_heap | Table | _pg_tde.pg_tdeam_handler | tde_heap table access method (9 rows) \dA+ * - List of access methods - Name | Type | Handler | Description -----------+-------+----------------------+---------------------------------------- - brin | Index | brinhandler | block range index (BRIN) access method - btree | Index | bthandler | b-tree index access method - gin | Index | ginhandler | GIN index access method - gist | Index | gisthandler | GiST index access method - hash | Index | hashhandler | hash index access method - heap | Table | heap_tableam_handler | heap table access method - heap2 | Table | heap_tableam_handler | - spgist | Index | spghandler | SP-GiST index access method - tde_heap | Table | pg_tdeam_handler | tde_heap table access method + List of access methods + Name | Type | Handler | Description +----------+-------+--------------------------+---------------------------------------- + brin | Index | brinhandler | block range index (BRIN) access method + btree | Index | bthandler | b-tree index access method + gin | Index | ginhandler | GIN index access method + gist | Index | gisthandler | GiST index access method + hash | Index | hashhandler | hash index access method + heap | Table | heap_tableam_handler | heap table access method + heap2 | Table | heap_tableam_handler | + spgist | Index | spghandler | SP-GiST index access method + tde_heap | Table | _pg_tde.pg_tdeam_handler | tde_heap table access method (9 rows) \dA+ h*