mirror of https://github.com/postgres/postgres
Make pg_waldump read encrypted WAL (#20)
This commit: - Adds build infrastructure for the frontend tools to compile with pg_tde. pg_tde dependencies will be built only if build flag `precone_ext` is on. It also makes `openssl` and `curl` required if `precone_ext` is on. - Makes pg_waldump work with the TDE encrypted WAL. If user set `-k` flag it will try to init tde keys etc and decode encrypted pages. If no `-k` options set, it behaves like community version - won't try to ini tde and won't be able to decrypt WAL. - Adds tap tests for encrypted WAL to pg_waldump. - Fixes Percona versioning in ./configure For PG-1003, PG-1005 Depends https://github.com/percona/pg_tde/pull/362pull/209/head
Andrew Pogrebnoi
10 months ago
committed by
GitHub
parent
5ad99f77be
commit
f4d272b6d8
@ -1 +1 @@ |
||||
Subproject commit ae29fd7e522deb54b2aa405ac9babc53576ac617 |
||||
Subproject commit e0978a8be6c70b2fccc86ca1cb8fc5499dd83a88 |
||||
@ -0,0 +1,247 @@ |
||||
|
||||
# Copyright (c) 2021-2024, PostgreSQL Global Development Group |
||||
|
||||
use strict; |
||||
use warnings FATAL => 'all'; |
||||
use PostgreSQL::Test::Cluster; |
||||
use PostgreSQL::Test::Utils; |
||||
use Test::More; |
||||
|
||||
my $node = PostgreSQL::Test::Cluster->new('main'); |
||||
$node->init; |
||||
$node->append_conf( |
||||
'postgresql.conf', q{ |
||||
autovacuum = off |
||||
checkpoint_timeout = 1h |
||||
|
||||
# for standbydesc |
||||
archive_mode=on |
||||
archive_command='' |
||||
|
||||
# for XLOG_HEAP_TRUNCATE |
||||
wal_level=logical |
||||
|
||||
# WAL Encryption |
||||
shared_preload_libraries = 'pg_tde' |
||||
pg_tde.wal_encrypt = on |
||||
}); |
||||
$node->start; |
||||
|
||||
my ($start_lsn, $start_walfile) = split /\|/, |
||||
$node->safe_psql('postgres', |
||||
q{SELECT pg_current_wal_insert_lsn(), pg_walfile_name(pg_current_wal_insert_lsn())} |
||||
); |
||||
|
||||
$node->safe_psql( |
||||
'postgres', q{ |
||||
-- heap, btree, hash, sequence |
||||
CREATE TABLE t1 (a int GENERATED ALWAYS AS IDENTITY, b text); |
||||
CREATE INDEX i1a ON t1 USING btree (a); |
||||
CREATE INDEX i1b ON t1 USING hash (b); |
||||
INSERT INTO t1 VALUES (default, 'one'), (default, 'two'); |
||||
DELETE FROM t1 WHERE b = 'one'; |
||||
TRUNCATE t1; |
||||
|
||||
-- abort |
||||
START TRANSACTION; |
||||
INSERT INTO t1 VALUES (default, 'three'); |
||||
ROLLBACK; |
||||
|
||||
-- unlogged/init fork |
||||
CREATE UNLOGGED TABLE t2 (x int); |
||||
CREATE INDEX i2 ON t2 USING btree (x); |
||||
INSERT INTO t2 SELECT generate_series(1, 10); |
||||
|
||||
-- gin |
||||
CREATE TABLE gin_idx_tbl (id bigserial PRIMARY KEY, data jsonb); |
||||
CREATE INDEX gin_idx ON gin_idx_tbl USING gin (data); |
||||
INSERT INTO gin_idx_tbl |
||||
WITH random_json AS ( |
||||
SELECT json_object_agg(key, trunc(random() * 10)) as json_data |
||||
FROM unnest(array['a', 'b', 'c']) as u(key)) |
||||
SELECT generate_series(1,500), json_data FROM random_json; |
||||
|
||||
-- gist, spgist |
||||
CREATE TABLE gist_idx_tbl (p point); |
||||
CREATE INDEX gist_idx ON gist_idx_tbl USING gist (p); |
||||
CREATE INDEX spgist_idx ON gist_idx_tbl USING spgist (p); |
||||
INSERT INTO gist_idx_tbl (p) VALUES (point '(1, 1)'), (point '(3, 2)'), (point '(6, 3)'); |
||||
|
||||
-- brin |
||||
CREATE TABLE brin_idx_tbl (col1 int, col2 text, col3 text ); |
||||
CREATE INDEX brin_idx ON brin_idx_tbl USING brin (col1, col2, col3) WITH (autosummarize=on); |
||||
INSERT INTO brin_idx_tbl SELECT generate_series(1, 10000), 'dummy', 'dummy'; |
||||
UPDATE brin_idx_tbl SET col2 = 'updated' WHERE col1 BETWEEN 1 AND 5000; |
||||
SELECT brin_summarize_range('brin_idx', 0); |
||||
SELECT brin_desummarize_range('brin_idx', 0); |
||||
|
||||
VACUUM; |
||||
|
||||
-- logical message |
||||
SELECT pg_logical_emit_message(true, 'foo', 'bar'); |
||||
|
||||
-- relmap |
||||
VACUUM FULL pg_authid; |
||||
|
||||
-- database |
||||
CREATE DATABASE d1; |
||||
DROP DATABASE d1; |
||||
}); |
||||
|
||||
my $tblspc_path = PostgreSQL::Test::Utils::tempdir_short(); |
||||
|
||||
$node->safe_psql( |
||||
'postgres', qq{ |
||||
CREATE TABLESPACE ts1 LOCATION '$tblspc_path'; |
||||
DROP TABLESPACE ts1; |
||||
}); |
||||
|
||||
my ($end_lsn, $end_walfile) = split /\|/, |
||||
$node->safe_psql('postgres', |
||||
q{SELECT pg_current_wal_insert_lsn(), pg_walfile_name(pg_current_wal_insert_lsn())} |
||||
); |
||||
|
||||
my $default_ts_oid = $node->safe_psql('postgres', |
||||
q{SELECT oid FROM pg_tablespace WHERE spcname = 'pg_default'}); |
||||
my $postgres_db_oid = $node->safe_psql('postgres', |
||||
q{SELECT oid FROM pg_database WHERE datname = 'postgres'}); |
||||
my $rel_t1_oid = $node->safe_psql('postgres', |
||||
q{SELECT oid FROM pg_class WHERE relname = 't1'}); |
||||
my $rel_i1a_oid = $node->safe_psql('postgres', |
||||
q{SELECT oid FROM pg_class WHERE relname = 'i1a'}); |
||||
|
||||
$node->stop; |
||||
|
||||
|
||||
# various ways of specifying WAL range |
||||
command_fails_like( |
||||
[ 'pg_waldump', 'foo', 'bar' ], |
||||
qr/error: could not locate WAL file "foo"/, |
||||
'start file not found'); |
||||
command_like([ 'pg_waldump', '-k', $node->data_dir. '/pg_tde', $node->data_dir . '/pg_wal/' . $start_walfile ], |
||||
qr/./, 'runs with start segment specified'); |
||||
command_fails_like( |
||||
[ 'pg_waldump', $node->data_dir . '/pg_wal/' . $start_walfile, 'bar' ], |
||||
qr/error: could not open file "bar"/, |
||||
'end file not found'); |
||||
command_like( |
||||
[ |
||||
'pg_waldump', |
||||
'-k', $node->data_dir. '/pg_tde', |
||||
$node->data_dir . '/pg_wal/' . $start_walfile, |
||||
$node->data_dir . '/pg_wal/' . $end_walfile |
||||
], |
||||
qr/./, |
||||
'runs with start and end segment specified'); |
||||
command_fails_like( |
||||
[ 'pg_waldump', '-p', $node->data_dir ], |
||||
qr/error: no start WAL location given/, |
||||
'path option requires start location'); |
||||
command_like( |
||||
[ |
||||
'pg_waldump', '-p', $node->data_dir, '--start', |
||||
$start_lsn, '--end', $end_lsn, |
||||
'-k', $node->data_dir. '/pg_tde' |
||||
], |
||||
qr/./, |
||||
'runs with path option and start and end locations'); |
||||
command_fails_like( |
||||
[ 'pg_waldump', '-k', $node->data_dir. '/pg_tde', '-p', $node->data_dir, '--start', $start_lsn ], |
||||
qr/error: error in WAL record at/, |
||||
'falling off the end of the WAL results in an error'); |
||||
|
||||
command_like( |
||||
[ |
||||
'pg_waldump', '--quiet', |
||||
'-k', $node->data_dir. '/pg_tde', |
||||
$node->data_dir . '/pg_wal/' . $start_walfile |
||||
], |
||||
qr/^$/, |
||||
'no output with --quiet option'); |
||||
command_fails_like( |
||||
[ 'pg_waldump', '--quiet', '-k', $node->data_dir. '/pg_tde', '-p', $node->data_dir, '--start', $start_lsn ], |
||||
qr/error: error in WAL record at/, |
||||
'errors are shown with --quiet'); |
||||
|
||||
|
||||
# Test for: Display a message that we're skipping data if `from` |
||||
# wasn't a pointer to the start of a record. |
||||
{ |
||||
# Construct a new LSN that is one byte past the original |
||||
# start_lsn. |
||||
my ($part1, $part2) = split qr{/}, $start_lsn; |
||||
my $lsn2 = hex $part2; |
||||
$lsn2++; |
||||
my $new_start = sprintf("%s/%X", $part1, $lsn2); |
||||
|
||||
my (@cmd, $stdout, $stderr, $result); |
||||
|
||||
@cmd = ( |
||||
'pg_waldump', '-k', $node->data_dir. '/pg_tde', |
||||
'--start', $new_start, |
||||
$node->data_dir . '/pg_wal/' . $start_walfile); |
||||
$result = IPC::Run::run \@cmd, '>', \$stdout, '2>', \$stderr; |
||||
ok($result, "runs with start segment and start LSN specified"); |
||||
like($stderr, qr/first record is after/, 'info message printed'); |
||||
} |
||||
|
||||
|
||||
# Helper function to test various options. Pass options as arguments. |
||||
# Output lines are returned as array. |
||||
sub test_pg_waldump |
||||
{ |
||||
local $Test::Builder::Level = $Test::Builder::Level + 1; |
||||
my @opts = @_; |
||||
|
||||
my (@cmd, $stdout, $stderr, $result, @lines); |
||||
|
||||
@cmd = ( |
||||
'pg_waldump', '-k', $node->data_dir. '/pg_tde', '-p', $node->data_dir, |
||||
'--start', $start_lsn, '--end', $end_lsn); |
||||
push @cmd, @opts; |
||||
$result = IPC::Run::run \@cmd, '>', \$stdout, '2>', \$stderr; |
||||
ok($result, "pg_waldump @opts: runs ok"); |
||||
is($stderr, '', "pg_waldump @opts: no stderr"); |
||||
@lines = split /\n/, $stdout; |
||||
ok(@lines > 0, "pg_waldump @opts: some lines are output"); |
||||
return @lines; |
||||
} |
||||
|
||||
my @lines; |
||||
|
||||
@lines = test_pg_waldump; |
||||
is(grep(!/^rmgr: \w/, @lines), 0, 'all output lines are rmgr lines'); |
||||
|
||||
@lines = test_pg_waldump('--limit', 6); |
||||
is(@lines, 6, 'limit option observed'); |
||||
|
||||
@lines = test_pg_waldump('--fullpage'); |
||||
is(grep(!/^rmgr:.*\bFPW\b/, @lines), 0, 'all output lines are FPW'); |
||||
|
||||
@lines = test_pg_waldump('--stats'); |
||||
like($lines[0], qr/WAL statistics/, "statistics on stdout"); |
||||
is(grep(/^rmgr:/, @lines), 0, 'no rmgr lines output'); |
||||
|
||||
@lines = test_pg_waldump('--stats=record'); |
||||
like($lines[0], qr/WAL statistics/, "statistics on stdout"); |
||||
is(grep(/^rmgr:/, @lines), 0, 'no rmgr lines output'); |
||||
|
||||
@lines = test_pg_waldump('--rmgr', 'Btree'); |
||||
is(grep(!/^rmgr: Btree/, @lines), 0, 'only Btree lines'); |
||||
|
||||
@lines = test_pg_waldump('--fork', 'init'); |
||||
is(grep(!/fork init/, @lines), 0, 'only init fork lines'); |
||||
|
||||
@lines = test_pg_waldump('--relation', |
||||
"$default_ts_oid/$postgres_db_oid/$rel_t1_oid"); |
||||
is(grep(!/rel $default_ts_oid\/$postgres_db_oid\/$rel_t1_oid/, @lines), |
||||
0, 'only lines for selected relation'); |
||||
|
||||
@lines = |
||||
test_pg_waldump('--relation', |
||||
"$default_ts_oid/$postgres_db_oid/$rel_i1a_oid", |
||||
'--block', 1); |
||||
is(grep(!/\bblk 1\b/, @lines), 0, 'only lines for selected block'); |
||||
|
||||
|
||||
done_testing(); |
||||
@ -0,0 +1,117 @@ |
||||
|
||||
# Copyright (c) 2022-2024, PostgreSQL Global Development Group |
||||
|
||||
use strict; |
||||
use warnings FATAL => 'all'; |
||||
use File::Basename; |
||||
use PostgreSQL::Test::Cluster; |
||||
use PostgreSQL::Test::RecursiveCopy; |
||||
use PostgreSQL::Test::Utils; |
||||
use Test::More; |
||||
|
||||
my ($blocksize, $walfile_name); |
||||
|
||||
# Function to extract the LSN from the given block structure |
||||
sub get_block_lsn |
||||
{ |
||||
my $path = shift; |
||||
my $blocksize = shift; |
||||
my $block; |
||||
|
||||
open my $fh, '<', $path or die "couldn't open file: $path\n"; |
||||
die "could not read block\n" |
||||
if $blocksize != read($fh, $block, $blocksize); |
||||
my ($lsn_hi, $lsn_lo) = unpack('LL', $block); |
||||
|
||||
$lsn_hi = sprintf('%08X', $lsn_hi); |
||||
$lsn_lo = sprintf('%08X', $lsn_lo); |
||||
|
||||
return ($lsn_hi, $lsn_lo); |
||||
} |
||||
|
||||
my $node = PostgreSQL::Test::Cluster->new('main'); |
||||
$node->init; |
||||
$node->append_conf( |
||||
'postgresql.conf', q{ |
||||
wal_level = 'replica' |
||||
max_wal_senders = 4 |
||||
|
||||
shared_preload_libraries = 'pg_tde' |
||||
pg_tde.wal_encrypt = on |
||||
}); |
||||
$node->start; |
||||
|
||||
# Generate data/WAL to examine that will have full pages in them. |
||||
$node->safe_psql( |
||||
'postgres', |
||||
"SELECT 'init' FROM pg_create_physical_replication_slot('regress_pg_waldump_slot', true, false); |
||||
CREATE TABLE test_table AS SELECT generate_series(1,100) a; |
||||
-- Force FPWs on the next writes. |
||||
CHECKPOINT; |
||||
UPDATE test_table SET a = a + 1; |
||||
"); |
||||
|
||||
($walfile_name, $blocksize) = split '\|' => $node->safe_psql('postgres', |
||||
"SELECT pg_walfile_name(pg_switch_wal()), current_setting('block_size')"); |
||||
|
||||
# Get the relation node, etc for the new table |
||||
my $relation = $node->safe_psql( |
||||
'postgres', |
||||
q{SELECT format( |
||||
'%s/%s/%s', |
||||
CASE WHEN reltablespace = 0 THEN dattablespace ELSE reltablespace END, |
||||
pg_database.oid, |
||||
pg_relation_filenode(pg_class.oid)) |
||||
FROM pg_class, pg_database |
||||
WHERE relname = 'test_table' AND |
||||
datname = current_database()} |
||||
); |
||||
|
||||
my $walfile = $node->data_dir . '/pg_wal/' . $walfile_name; |
||||
my $tmp_folder = PostgreSQL::Test::Utils::tempdir; |
||||
|
||||
ok(-f $walfile, "Got a WAL file"); |
||||
|
||||
$node->command_ok( |
||||
[ |
||||
'pg_waldump', '--quiet', |
||||
'-k', $node->data_dir. '/pg_tde', |
||||
'--save-fullpage', "$tmp_folder/raw", |
||||
'--relation', $relation, |
||||
$walfile |
||||
], |
||||
'pg_waldump with --save-fullpage runs'); |
||||
|
||||
# This regexp will match filenames formatted as: |
||||
# TLI-LSNh-LSNl.TBLSPCOID.DBOID.NODEOID.dd_fork with the components being: |
||||
# - Timeline ID in hex format. |
||||
# - WAL LSN in hex format, as two 8-character numbers. |
||||
# - Tablespace OID (0 for global). |
||||
# - Database OID. |
||||
# - Relfilenode. |
||||
# - Block number. |
||||
# - Fork this block came from (vm, init, fsm, or main). |
||||
my $file_re = |
||||
qr/^[0-9A-F]{8}-([0-9A-F]{8})-([0-9A-F]{8})[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+(?:_vm|_init|_fsm|_main)?$/; |
||||
|
||||
my $file_count = 0; |
||||
|
||||
# Verify filename format matches --save-fullpage. |
||||
for my $fullpath (glob "$tmp_folder/raw/*") |
||||
{ |
||||
my $file = File::Basename::basename($fullpath); |
||||
|
||||
like($file, $file_re, "verify filename format for file $file"); |
||||
$file_count++; |
||||
|
||||
my ($hi_lsn_fn, $lo_lsn_fn) = ($file =~ $file_re); |
||||
my ($hi_lsn_bk, $lo_lsn_bk) = get_block_lsn($fullpath, $blocksize); |
||||
|
||||
# The LSN on the block comes before the file's LSN. |
||||
ok( $hi_lsn_fn . $lo_lsn_fn gt $hi_lsn_bk . $lo_lsn_bk, |
||||
'LSN stored in the file precedes the one stored in the block'); |
||||
} |
||||
|
||||
ok($file_count > 0, 'verify that at least one block has been saved'); |
||||
|
||||
done_testing(); |
||||
Loading…
Reference in new issue