mirror of https://github.com/postgres/postgres
Make pg_waldump read encrypted WAL (#20)
This commit: - Adds build infrastructure for the frontend tools to compile with pg_tde. pg_tde dependencies will be built only if build flag `precone_ext` is on. It also makes `openssl` and `curl` required if `precone_ext` is on. - Makes pg_waldump work with the TDE encrypted WAL. If user set `-k` flag it will try to init tde keys etc and decode encrypted pages. If no `-k` options set, it behaves like community version - won't try to ini tde and won't be able to decrypt WAL. - Adds tap tests for encrypted WAL to pg_waldump. - Fixes Percona versioning in ./configure For PG-1003, PG-1005 Depends https://github.com/percona/pg_tde/pull/362pull/209/head
Andrew Pogrebnoi
10 months ago
committed by
GitHub
parent
5ad99f77be
commit
f4d272b6d8
@ -1 +1 @@ |
|||||||
Subproject commit ae29fd7e522deb54b2aa405ac9babc53576ac617 |
Subproject commit e0978a8be6c70b2fccc86ca1cb8fc5499dd83a88 |
||||||
@ -0,0 +1,247 @@ |
|||||||
|
|
||||||
|
# Copyright (c) 2021-2024, PostgreSQL Global Development Group |
||||||
|
|
||||||
|
use strict; |
||||||
|
use warnings FATAL => 'all'; |
||||||
|
use PostgreSQL::Test::Cluster; |
||||||
|
use PostgreSQL::Test::Utils; |
||||||
|
use Test::More; |
||||||
|
|
||||||
|
my $node = PostgreSQL::Test::Cluster->new('main'); |
||||||
|
$node->init; |
||||||
|
$node->append_conf( |
||||||
|
'postgresql.conf', q{ |
||||||
|
autovacuum = off |
||||||
|
checkpoint_timeout = 1h |
||||||
|
|
||||||
|
# for standbydesc |
||||||
|
archive_mode=on |
||||||
|
archive_command='' |
||||||
|
|
||||||
|
# for XLOG_HEAP_TRUNCATE |
||||||
|
wal_level=logical |
||||||
|
|
||||||
|
# WAL Encryption |
||||||
|
shared_preload_libraries = 'pg_tde' |
||||||
|
pg_tde.wal_encrypt = on |
||||||
|
}); |
||||||
|
$node->start; |
||||||
|
|
||||||
|
my ($start_lsn, $start_walfile) = split /\|/, |
||||||
|
$node->safe_psql('postgres', |
||||||
|
q{SELECT pg_current_wal_insert_lsn(), pg_walfile_name(pg_current_wal_insert_lsn())} |
||||||
|
); |
||||||
|
|
||||||
|
$node->safe_psql( |
||||||
|
'postgres', q{ |
||||||
|
-- heap, btree, hash, sequence |
||||||
|
CREATE TABLE t1 (a int GENERATED ALWAYS AS IDENTITY, b text); |
||||||
|
CREATE INDEX i1a ON t1 USING btree (a); |
||||||
|
CREATE INDEX i1b ON t1 USING hash (b); |
||||||
|
INSERT INTO t1 VALUES (default, 'one'), (default, 'two'); |
||||||
|
DELETE FROM t1 WHERE b = 'one'; |
||||||
|
TRUNCATE t1; |
||||||
|
|
||||||
|
-- abort |
||||||
|
START TRANSACTION; |
||||||
|
INSERT INTO t1 VALUES (default, 'three'); |
||||||
|
ROLLBACK; |
||||||
|
|
||||||
|
-- unlogged/init fork |
||||||
|
CREATE UNLOGGED TABLE t2 (x int); |
||||||
|
CREATE INDEX i2 ON t2 USING btree (x); |
||||||
|
INSERT INTO t2 SELECT generate_series(1, 10); |
||||||
|
|
||||||
|
-- gin |
||||||
|
CREATE TABLE gin_idx_tbl (id bigserial PRIMARY KEY, data jsonb); |
||||||
|
CREATE INDEX gin_idx ON gin_idx_tbl USING gin (data); |
||||||
|
INSERT INTO gin_idx_tbl |
||||||
|
WITH random_json AS ( |
||||||
|
SELECT json_object_agg(key, trunc(random() * 10)) as json_data |
||||||
|
FROM unnest(array['a', 'b', 'c']) as u(key)) |
||||||
|
SELECT generate_series(1,500), json_data FROM random_json; |
||||||
|
|
||||||
|
-- gist, spgist |
||||||
|
CREATE TABLE gist_idx_tbl (p point); |
||||||
|
CREATE INDEX gist_idx ON gist_idx_tbl USING gist (p); |
||||||
|
CREATE INDEX spgist_idx ON gist_idx_tbl USING spgist (p); |
||||||
|
INSERT INTO gist_idx_tbl (p) VALUES (point '(1, 1)'), (point '(3, 2)'), (point '(6, 3)'); |
||||||
|
|
||||||
|
-- brin |
||||||
|
CREATE TABLE brin_idx_tbl (col1 int, col2 text, col3 text ); |
||||||
|
CREATE INDEX brin_idx ON brin_idx_tbl USING brin (col1, col2, col3) WITH (autosummarize=on); |
||||||
|
INSERT INTO brin_idx_tbl SELECT generate_series(1, 10000), 'dummy', 'dummy'; |
||||||
|
UPDATE brin_idx_tbl SET col2 = 'updated' WHERE col1 BETWEEN 1 AND 5000; |
||||||
|
SELECT brin_summarize_range('brin_idx', 0); |
||||||
|
SELECT brin_desummarize_range('brin_idx', 0); |
||||||
|
|
||||||
|
VACUUM; |
||||||
|
|
||||||
|
-- logical message |
||||||
|
SELECT pg_logical_emit_message(true, 'foo', 'bar'); |
||||||
|
|
||||||
|
-- relmap |
||||||
|
VACUUM FULL pg_authid; |
||||||
|
|
||||||
|
-- database |
||||||
|
CREATE DATABASE d1; |
||||||
|
DROP DATABASE d1; |
||||||
|
}); |
||||||
|
|
||||||
|
my $tblspc_path = PostgreSQL::Test::Utils::tempdir_short(); |
||||||
|
|
||||||
|
$node->safe_psql( |
||||||
|
'postgres', qq{ |
||||||
|
CREATE TABLESPACE ts1 LOCATION '$tblspc_path'; |
||||||
|
DROP TABLESPACE ts1; |
||||||
|
}); |
||||||
|
|
||||||
|
my ($end_lsn, $end_walfile) = split /\|/, |
||||||
|
$node->safe_psql('postgres', |
||||||
|
q{SELECT pg_current_wal_insert_lsn(), pg_walfile_name(pg_current_wal_insert_lsn())} |
||||||
|
); |
||||||
|
|
||||||
|
my $default_ts_oid = $node->safe_psql('postgres', |
||||||
|
q{SELECT oid FROM pg_tablespace WHERE spcname = 'pg_default'}); |
||||||
|
my $postgres_db_oid = $node->safe_psql('postgres', |
||||||
|
q{SELECT oid FROM pg_database WHERE datname = 'postgres'}); |
||||||
|
my $rel_t1_oid = $node->safe_psql('postgres', |
||||||
|
q{SELECT oid FROM pg_class WHERE relname = 't1'}); |
||||||
|
my $rel_i1a_oid = $node->safe_psql('postgres', |
||||||
|
q{SELECT oid FROM pg_class WHERE relname = 'i1a'}); |
||||||
|
|
||||||
|
$node->stop; |
||||||
|
|
||||||
|
|
||||||
|
# various ways of specifying WAL range |
||||||
|
command_fails_like( |
||||||
|
[ 'pg_waldump', 'foo', 'bar' ], |
||||||
|
qr/error: could not locate WAL file "foo"/, |
||||||
|
'start file not found'); |
||||||
|
command_like([ 'pg_waldump', '-k', $node->data_dir. '/pg_tde', $node->data_dir . '/pg_wal/' . $start_walfile ], |
||||||
|
qr/./, 'runs with start segment specified'); |
||||||
|
command_fails_like( |
||||||
|
[ 'pg_waldump', $node->data_dir . '/pg_wal/' . $start_walfile, 'bar' ], |
||||||
|
qr/error: could not open file "bar"/, |
||||||
|
'end file not found'); |
||||||
|
command_like( |
||||||
|
[ |
||||||
|
'pg_waldump', |
||||||
|
'-k', $node->data_dir. '/pg_tde', |
||||||
|
$node->data_dir . '/pg_wal/' . $start_walfile, |
||||||
|
$node->data_dir . '/pg_wal/' . $end_walfile |
||||||
|
], |
||||||
|
qr/./, |
||||||
|
'runs with start and end segment specified'); |
||||||
|
command_fails_like( |
||||||
|
[ 'pg_waldump', '-p', $node->data_dir ], |
||||||
|
qr/error: no start WAL location given/, |
||||||
|
'path option requires start location'); |
||||||
|
command_like( |
||||||
|
[ |
||||||
|
'pg_waldump', '-p', $node->data_dir, '--start', |
||||||
|
$start_lsn, '--end', $end_lsn, |
||||||
|
'-k', $node->data_dir. '/pg_tde' |
||||||
|
], |
||||||
|
qr/./, |
||||||
|
'runs with path option and start and end locations'); |
||||||
|
command_fails_like( |
||||||
|
[ 'pg_waldump', '-k', $node->data_dir. '/pg_tde', '-p', $node->data_dir, '--start', $start_lsn ], |
||||||
|
qr/error: error in WAL record at/, |
||||||
|
'falling off the end of the WAL results in an error'); |
||||||
|
|
||||||
|
command_like( |
||||||
|
[ |
||||||
|
'pg_waldump', '--quiet', |
||||||
|
'-k', $node->data_dir. '/pg_tde', |
||||||
|
$node->data_dir . '/pg_wal/' . $start_walfile |
||||||
|
], |
||||||
|
qr/^$/, |
||||||
|
'no output with --quiet option'); |
||||||
|
command_fails_like( |
||||||
|
[ 'pg_waldump', '--quiet', '-k', $node->data_dir. '/pg_tde', '-p', $node->data_dir, '--start', $start_lsn ], |
||||||
|
qr/error: error in WAL record at/, |
||||||
|
'errors are shown with --quiet'); |
||||||
|
|
||||||
|
|
||||||
|
# Test for: Display a message that we're skipping data if `from` |
||||||
|
# wasn't a pointer to the start of a record. |
||||||
|
{ |
||||||
|
# Construct a new LSN that is one byte past the original |
||||||
|
# start_lsn. |
||||||
|
my ($part1, $part2) = split qr{/}, $start_lsn; |
||||||
|
my $lsn2 = hex $part2; |
||||||
|
$lsn2++; |
||||||
|
my $new_start = sprintf("%s/%X", $part1, $lsn2); |
||||||
|
|
||||||
|
my (@cmd, $stdout, $stderr, $result); |
||||||
|
|
||||||
|
@cmd = ( |
||||||
|
'pg_waldump', '-k', $node->data_dir. '/pg_tde', |
||||||
|
'--start', $new_start, |
||||||
|
$node->data_dir . '/pg_wal/' . $start_walfile); |
||||||
|
$result = IPC::Run::run \@cmd, '>', \$stdout, '2>', \$stderr; |
||||||
|
ok($result, "runs with start segment and start LSN specified"); |
||||||
|
like($stderr, qr/first record is after/, 'info message printed'); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
# Helper function to test various options. Pass options as arguments. |
||||||
|
# Output lines are returned as array. |
||||||
|
sub test_pg_waldump |
||||||
|
{ |
||||||
|
local $Test::Builder::Level = $Test::Builder::Level + 1; |
||||||
|
my @opts = @_; |
||||||
|
|
||||||
|
my (@cmd, $stdout, $stderr, $result, @lines); |
||||||
|
|
||||||
|
@cmd = ( |
||||||
|
'pg_waldump', '-k', $node->data_dir. '/pg_tde', '-p', $node->data_dir, |
||||||
|
'--start', $start_lsn, '--end', $end_lsn); |
||||||
|
push @cmd, @opts; |
||||||
|
$result = IPC::Run::run \@cmd, '>', \$stdout, '2>', \$stderr; |
||||||
|
ok($result, "pg_waldump @opts: runs ok"); |
||||||
|
is($stderr, '', "pg_waldump @opts: no stderr"); |
||||||
|
@lines = split /\n/, $stdout; |
||||||
|
ok(@lines > 0, "pg_waldump @opts: some lines are output"); |
||||||
|
return @lines; |
||||||
|
} |
||||||
|
|
||||||
|
my @lines; |
||||||
|
|
||||||
|
@lines = test_pg_waldump; |
||||||
|
is(grep(!/^rmgr: \w/, @lines), 0, 'all output lines are rmgr lines'); |
||||||
|
|
||||||
|
@lines = test_pg_waldump('--limit', 6); |
||||||
|
is(@lines, 6, 'limit option observed'); |
||||||
|
|
||||||
|
@lines = test_pg_waldump('--fullpage'); |
||||||
|
is(grep(!/^rmgr:.*\bFPW\b/, @lines), 0, 'all output lines are FPW'); |
||||||
|
|
||||||
|
@lines = test_pg_waldump('--stats'); |
||||||
|
like($lines[0], qr/WAL statistics/, "statistics on stdout"); |
||||||
|
is(grep(/^rmgr:/, @lines), 0, 'no rmgr lines output'); |
||||||
|
|
||||||
|
@lines = test_pg_waldump('--stats=record'); |
||||||
|
like($lines[0], qr/WAL statistics/, "statistics on stdout"); |
||||||
|
is(grep(/^rmgr:/, @lines), 0, 'no rmgr lines output'); |
||||||
|
|
||||||
|
@lines = test_pg_waldump('--rmgr', 'Btree'); |
||||||
|
is(grep(!/^rmgr: Btree/, @lines), 0, 'only Btree lines'); |
||||||
|
|
||||||
|
@lines = test_pg_waldump('--fork', 'init'); |
||||||
|
is(grep(!/fork init/, @lines), 0, 'only init fork lines'); |
||||||
|
|
||||||
|
@lines = test_pg_waldump('--relation', |
||||||
|
"$default_ts_oid/$postgres_db_oid/$rel_t1_oid"); |
||||||
|
is(grep(!/rel $default_ts_oid\/$postgres_db_oid\/$rel_t1_oid/, @lines), |
||||||
|
0, 'only lines for selected relation'); |
||||||
|
|
||||||
|
@lines = |
||||||
|
test_pg_waldump('--relation', |
||||||
|
"$default_ts_oid/$postgres_db_oid/$rel_i1a_oid", |
||||||
|
'--block', 1); |
||||||
|
is(grep(!/\bblk 1\b/, @lines), 0, 'only lines for selected block'); |
||||||
|
|
||||||
|
|
||||||
|
done_testing(); |
||||||
@ -0,0 +1,117 @@ |
|||||||
|
|
||||||
|
# Copyright (c) 2022-2024, PostgreSQL Global Development Group |
||||||
|
|
||||||
|
use strict; |
||||||
|
use warnings FATAL => 'all'; |
||||||
|
use File::Basename; |
||||||
|
use PostgreSQL::Test::Cluster; |
||||||
|
use PostgreSQL::Test::RecursiveCopy; |
||||||
|
use PostgreSQL::Test::Utils; |
||||||
|
use Test::More; |
||||||
|
|
||||||
|
my ($blocksize, $walfile_name); |
||||||
|
|
||||||
|
# Function to extract the LSN from the given block structure |
||||||
|
sub get_block_lsn |
||||||
|
{ |
||||||
|
my $path = shift; |
||||||
|
my $blocksize = shift; |
||||||
|
my $block; |
||||||
|
|
||||||
|
open my $fh, '<', $path or die "couldn't open file: $path\n"; |
||||||
|
die "could not read block\n" |
||||||
|
if $blocksize != read($fh, $block, $blocksize); |
||||||
|
my ($lsn_hi, $lsn_lo) = unpack('LL', $block); |
||||||
|
|
||||||
|
$lsn_hi = sprintf('%08X', $lsn_hi); |
||||||
|
$lsn_lo = sprintf('%08X', $lsn_lo); |
||||||
|
|
||||||
|
return ($lsn_hi, $lsn_lo); |
||||||
|
} |
||||||
|
|
||||||
|
my $node = PostgreSQL::Test::Cluster->new('main'); |
||||||
|
$node->init; |
||||||
|
$node->append_conf( |
||||||
|
'postgresql.conf', q{ |
||||||
|
wal_level = 'replica' |
||||||
|
max_wal_senders = 4 |
||||||
|
|
||||||
|
shared_preload_libraries = 'pg_tde' |
||||||
|
pg_tde.wal_encrypt = on |
||||||
|
}); |
||||||
|
$node->start; |
||||||
|
|
||||||
|
# Generate data/WAL to examine that will have full pages in them. |
||||||
|
$node->safe_psql( |
||||||
|
'postgres', |
||||||
|
"SELECT 'init' FROM pg_create_physical_replication_slot('regress_pg_waldump_slot', true, false); |
||||||
|
CREATE TABLE test_table AS SELECT generate_series(1,100) a; |
||||||
|
-- Force FPWs on the next writes. |
||||||
|
CHECKPOINT; |
||||||
|
UPDATE test_table SET a = a + 1; |
||||||
|
"); |
||||||
|
|
||||||
|
($walfile_name, $blocksize) = split '\|' => $node->safe_psql('postgres', |
||||||
|
"SELECT pg_walfile_name(pg_switch_wal()), current_setting('block_size')"); |
||||||
|
|
||||||
|
# Get the relation node, etc for the new table |
||||||
|
my $relation = $node->safe_psql( |
||||||
|
'postgres', |
||||||
|
q{SELECT format( |
||||||
|
'%s/%s/%s', |
||||||
|
CASE WHEN reltablespace = 0 THEN dattablespace ELSE reltablespace END, |
||||||
|
pg_database.oid, |
||||||
|
pg_relation_filenode(pg_class.oid)) |
||||||
|
FROM pg_class, pg_database |
||||||
|
WHERE relname = 'test_table' AND |
||||||
|
datname = current_database()} |
||||||
|
); |
||||||
|
|
||||||
|
my $walfile = $node->data_dir . '/pg_wal/' . $walfile_name; |
||||||
|
my $tmp_folder = PostgreSQL::Test::Utils::tempdir; |
||||||
|
|
||||||
|
ok(-f $walfile, "Got a WAL file"); |
||||||
|
|
||||||
|
$node->command_ok( |
||||||
|
[ |
||||||
|
'pg_waldump', '--quiet', |
||||||
|
'-k', $node->data_dir. '/pg_tde', |
||||||
|
'--save-fullpage', "$tmp_folder/raw", |
||||||
|
'--relation', $relation, |
||||||
|
$walfile |
||||||
|
], |
||||||
|
'pg_waldump with --save-fullpage runs'); |
||||||
|
|
||||||
|
# This regexp will match filenames formatted as: |
||||||
|
# TLI-LSNh-LSNl.TBLSPCOID.DBOID.NODEOID.dd_fork with the components being: |
||||||
|
# - Timeline ID in hex format. |
||||||
|
# - WAL LSN in hex format, as two 8-character numbers. |
||||||
|
# - Tablespace OID (0 for global). |
||||||
|
# - Database OID. |
||||||
|
# - Relfilenode. |
||||||
|
# - Block number. |
||||||
|
# - Fork this block came from (vm, init, fsm, or main). |
||||||
|
my $file_re = |
||||||
|
qr/^[0-9A-F]{8}-([0-9A-F]{8})-([0-9A-F]{8})[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+(?:_vm|_init|_fsm|_main)?$/; |
||||||
|
|
||||||
|
my $file_count = 0; |
||||||
|
|
||||||
|
# Verify filename format matches --save-fullpage. |
||||||
|
for my $fullpath (glob "$tmp_folder/raw/*") |
||||||
|
{ |
||||||
|
my $file = File::Basename::basename($fullpath); |
||||||
|
|
||||||
|
like($file, $file_re, "verify filename format for file $file"); |
||||||
|
$file_count++; |
||||||
|
|
||||||
|
my ($hi_lsn_fn, $lo_lsn_fn) = ($file =~ $file_re); |
||||||
|
my ($hi_lsn_bk, $lo_lsn_bk) = get_block_lsn($fullpath, $blocksize); |
||||||
|
|
||||||
|
# The LSN on the block comes before the file's LSN. |
||||||
|
ok( $hi_lsn_fn . $lo_lsn_fn gt $hi_lsn_bk . $lo_lsn_bk, |
||||||
|
'LSN stored in the file precedes the one stored in the block'); |
||||||
|
} |
||||||
|
|
||||||
|
ok($file_count > 0, 'verify that at least one block has been saved'); |
||||||
|
|
||||||
|
done_testing(); |
||||||
Loading…
Reference in new issue