open-dsi
/
postgres
mirror of https://github.com/postgres/postgres
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move routines to manipulate WAL into PostgreSQL::Test::Cluster

Browse Source 
These facilities were originally in the recovery TAP test
039_end_of_wal.pl.  A follow-up bug fix with a TAP test doing similar
WAL manipulations requires them, and all these had better not be
duplicated due to their complexity.  The routine names are tweaked to
use "wal" more consistently, similarly to the existing "advance_wal".

In v14 and v13, the new routines are moved to PostgresNode.pm.
039_end_of_wal.pl is updated to use the refactored routines, without
changing its coverage.

Reviewed-by: Alexander Kukushkin
Discussion: https://postgr.es/m/CAFh8B=mozC+e1wGJq0H=0O65goZju+6ab5AU7DEWCSUA2OtwDg@mail.gmail.com
Backpatch-through: 13
REL_16_STABLE
Michael Paquier 8 months ago
parent b935691b8b
commit 9420f9bb61
2 changed files with 214 additions and 191 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 148
      src/test/perl/PostgreSQL/Test/Cluster.pm
  2. 257
      src/test/recovery/t/039_end_of_wal.pl

148
src/test/perl/PostgreSQL/Test/Cluster.pm
Unescape View File

@ -2660,6 +2660,154 @@ sub lsn
=pod
=item $node->write_wal($tli, $lsn, $segment_size, $data)
Write some arbitrary data in WAL for the given segment at $lsn (in bytes).
This should be called while the cluster is not running.
Returns the path of the WAL segment written to.
=cut
sub write_wal
{
my ($self, $tli, $lsn, $segment_size, $data) = @_;
# Calculate segment number and offset position in segment based on the
# input LSN.
my $segment = $lsn / $segment_size;
my $offset = $lsn % $segment_size;
my $path =
sprintf("%s/pg_wal/%08X%08X%08X", $self->data_dir, $tli, 0, $segment);
open my $fh, "+<:raw", $path or die "could not open WAL segment $path";
seek($fh, $offset, SEEK_SET) or die "could not seek WAL segment $path";
print $fh $data;
close $fh;
return $path;
}
=pod
=item $node->emit_wal($size)
Emit a WAL record of arbitrary size, using pg_logical_emit_message().
Returns the end LSN of the record inserted, in bytes.
=cut
sub emit_wal
{
my ($self, $size) = @_;
return int(
$self->safe_psql(
'postgres',
"SELECT pg_logical_emit_message(true, '', repeat('a', $size)) - '0/0'"
));
}
# Private routine returning the current insert LSN of a node, in bytes.
# Used by the routines below in charge of advancing WAL to arbitrary
# positions. The insert LSN is returned in bytes.
sub _get_insert_lsn
{
my ($self) = @_;
return int(
$self->safe_psql(
'postgres', "SELECT pg_current_wal_insert_lsn() - '0/0'"));
}
=pod
=item $node->advance_wal_out_of_record_splitting_zone($wal_block_size)
Advance WAL at the end of a page, making sure that we are far away enough
from the end of a page that we could insert a couple of small records.
This inserts a few records of a fixed size, until the threshold gets close
enough to the end of the WAL page inserting records to.
Returns the end LSN up to which WAL has advanced, in bytes.
=cut
sub advance_wal_out_of_record_splitting_zone
{
my ($self, $wal_block_size) = @_;
my $page_threshold = $wal_block_size / 4;
my $end_lsn = $self->_get_insert_lsn();
my $page_offset = $end_lsn % $wal_block_size;
while ($page_offset >= $wal_block_size - $page_threshold)
{
$self->emit_wal($page_threshold);
$end_lsn = $self->_get_insert_lsn();
$page_offset = $end_lsn % $wal_block_size;
}
return $end_lsn;
}
=pod
=item $node->advance_wal_to_record_splitting_zone($wal_block_size)
Advance WAL so close to the end of a page that an XLogRecordHeader would not
fit on it.
Returns the end LSN up to which WAL has advanced, in bytes.
=cut
sub advance_wal_to_record_splitting_zone
{
my ($self, $wal_block_size) = @_;
# Size of record header.
my $RECORD_HEADER_SIZE = 24;
my $end_lsn = $self->_get_insert_lsn();
my $page_offset = $end_lsn % $wal_block_size;
# Get fairly close to the end of a page in big steps
while ($page_offset <= $wal_block_size - 512)
{
$self->emit_wal($wal_block_size - $page_offset - 256);
$end_lsn = $self->_get_insert_lsn();
$page_offset = $end_lsn % $wal_block_size;
}
# Calibrate our message size so that we can get closer 8 bytes at
# a time.
my $message_size = $wal_block_size - 80;
while ($page_offset <= $wal_block_size - $RECORD_HEADER_SIZE)
{
$self->emit_wal($message_size);
$end_lsn = $self->_get_insert_lsn();
my $old_offset = $page_offset;
$page_offset = $end_lsn % $wal_block_size;
# Adjust the message size until it causes 8 bytes changes in
# offset, enough to be able to split a record header.
my $delta = $page_offset - $old_offset;
if ($delta > 8)
{
$message_size -= 8;
}
elsif ($delta <= 0)
{
$message_size += 8;
}
}
return $end_lsn;
}
=pod
=item $node->wait_for_catchup(standby_name, mode, target_lsn)
Wait for the replication connection with application_name standby_name until

257
src/test/recovery/t/039_end_of_wal.pl
Unescape View File

@ -20,9 +20,6 @@ use integer; # causes / operator to use integer math
# we need to know the endianness to do that.
my $BIG_ENDIAN = pack("L", 0x12345678) eq pack("N", 0x12345678);
# Header size of record header.
my $RECORD_HEADER_SIZE = 24;
# Fields retrieved from code headers.
my @scan_result = scan_server_header('access/xlog_internal.h',
'#define\s+XLOG_PAGE_MAGIC\s+(\w+)');
@ -36,64 +33,6 @@ my $WAL_SEGMENT_SIZE;
my $WAL_BLOCK_SIZE;
my $TLI;
# Build path of a WAL segment.
sub wal_segment_path
{
my $node = shift;
my $tli = shift;
my $segment = shift;
my $wal_path =
sprintf("%s/pg_wal/%08X%08X%08X", $node->data_dir, $tli, 0, $segment);
return $wal_path;
}
# Calculate from a LSN (in bytes) its segment number and its offset.
sub lsn_to_segment_and_offset
{
my $lsn = shift;
return ($lsn / $WAL_SEGMENT_SIZE, $lsn % $WAL_SEGMENT_SIZE);
}
# Write some arbitrary data in WAL for the given segment at LSN.
# This should be called while the cluster is not running.
sub write_wal
{
my $node = shift;
my $tli = shift;
my $lsn = shift;
my $data = shift;
my ($segment, $offset) = lsn_to_segment_and_offset($lsn);
my $path = wal_segment_path($node, $tli, $segment);
open my $fh, "+<:raw", $path or die;
seek($fh, $offset, SEEK_SET) or die;
print $fh $data;
close $fh;
}
# Emit a WAL record of arbitrary size. Returns the end LSN of the
# record inserted, in bytes.
sub emit_message
{
my $node = shift;
my $size = shift;
return int(
$node->safe_psql(
'postgres',
"SELECT pg_logical_emit_message(true, '', repeat('a', $size)) - '0/0'"
));
}
# Get the current insert LSN of a node, in bytes.
sub get_insert_lsn
{
my $node = shift;
return int(
$node->safe_psql(
'postgres', "SELECT pg_current_wal_insert_lsn() - '0/0'"));
}
# Get GUC value, converted to an int.
sub get_int_setting
{
@ -167,69 +106,6 @@ sub build_page_header
$BIG_ENDIAN ? $xlp_pageaddr : 0, $xlp_rem_len);
}
# Make sure we are far away enough from the end of a page that we could insert
# a couple of small records. This inserts a few records of a fixed size, until
# the threshold gets close enough to the end of the WAL page inserting records
# to.
sub advance_out_of_record_splitting_zone
{
my $node = shift;
my $page_threshold = $WAL_BLOCK_SIZE / 4;
my $end_lsn = get_insert_lsn($node);
my $page_offset = $end_lsn % $WAL_BLOCK_SIZE;
while ($page_offset >= $WAL_BLOCK_SIZE - $page_threshold)
{
emit_message($node, $page_threshold);
$end_lsn = get_insert_lsn($node);
$page_offset = $end_lsn % $WAL_BLOCK_SIZE;
}
return $end_lsn;
}
# Advance so close to the end of a page that an XLogRecordHeader would not
# fit on it.
sub advance_to_record_splitting_zone
{
my $node = shift;
my $end_lsn = get_insert_lsn($node);
my $page_offset = $end_lsn % $WAL_BLOCK_SIZE;
# Get fairly close to the end of a page in big steps
while ($page_offset <= $WAL_BLOCK_SIZE - 512)
{
emit_message($node, $WAL_BLOCK_SIZE - $page_offset - 256);
$end_lsn = get_insert_lsn($node);
$page_offset = $end_lsn % $WAL_BLOCK_SIZE;
}
# Calibrate our message size so that we can get closer 8 bytes at
# a time.
my $message_size = $WAL_BLOCK_SIZE - 80;
while ($page_offset <= $WAL_BLOCK_SIZE - $RECORD_HEADER_SIZE)
{
emit_message($node, $message_size);
$end_lsn = get_insert_lsn($node);
my $old_offset = $page_offset;
$page_offset = $end_lsn % $WAL_BLOCK_SIZE;
# Adjust the message size until it causes 8 bytes changes in
# offset, enough to be able to split a record header.
my $delta = $page_offset - $old_offset;
if ($delta > 8)
{
$message_size -= 8;
}
elsif ($delta <= 0)
{
$message_size += 8;
}
}
return $end_lsn;
}
# Setup a new node. The configuration chosen here minimizes the number
# of arbitrary records that could get generated in a cluster. Enlarging
# checkpoint_timeout avoids noise with checkpoint activity. wal_level
@ -265,8 +141,8 @@ note "Single-page end-of-WAL detection";
###########################################################################
# xl_tot_len is 0 (a common case, we hit trailing zeroes).
emit_message($node, 0);
$end_lsn = advance_out_of_record_splitting_zone($node);
$node->emit_wal(0);
$end_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
$node->stop('immediate');
my $log_size = -s $node->logfile;
$node->start;
@ -276,10 +152,10 @@ ok( $node->log_contains(
"xl_tot_len zero");
# xl_tot_len is < 24 (presumably recycled garbage).
emit_message($node, 0);
$end_lsn = advance_out_of_record_splitting_zone($node);
$node->emit_wal(0);
$end_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
$node->stop('immediate');
write_wal($node, $TLI, $end_lsn, build_record_header(23));
$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE, build_record_header(23));
$log_size = -s $node->logfile;
$node->start;
ok( $node->log_contains(
@ -289,10 +165,10 @@ ok( $node->log_contains(
# xl_tot_len in final position, not big enough to span into a new page but
# also not eligible for regular record header validation
emit_message($node, 0);
$end_lsn = advance_to_record_splitting_zone($node);
$node->emit_wal(0);
$end_lsn = $node->advance_wal_to_record_splitting_zone($WAL_BLOCK_SIZE);
$node->stop('immediate');
write_wal($node, $TLI, $end_lsn, build_record_header(1));
$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE, build_record_header(1));
$log_size = -s $node->logfile;
$node->start;
ok( $node->log_contains(
@ -301,10 +177,10 @@ ok( $node->log_contains(
"xl_tot_len short at end-of-page");
# Need more pages, but xl_prev check fails first.
emit_message($node, 0);
$end_lsn = advance_out_of_record_splitting_zone($node);
$node->emit_wal(0);
$end_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
$node->stop('immediate');
write_wal($node, $TLI, $end_lsn,
$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef));
$log_size = -s $node->logfile;
$node->start;
@ -313,12 +189,12 @@ ok( $node->log_contains(
"xl_prev bad");
# xl_crc check fails.
emit_message($node, 0);
advance_out_of_record_splitting_zone($node);
$end_lsn = emit_message($node, 10);
$node->emit_wal(0);
$node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
$end_lsn = $node->emit_wal(10);
$node->stop('immediate');
# Corrupt a byte in that record, breaking its CRC.
write_wal($node, $TLI, $end_lsn - 8, '!');
$node->write_wal($TLI, $end_lsn - 8, $WAL_SEGMENT_SIZE, '!');
$log_size = -s $node->logfile;
$node->start;
ok( $node->log_contains(
@ -335,11 +211,11 @@ note "Multi-page end-of-WAL detection, header is not split";
# written to WAL.
# Good xl_prev, we hit zero page next (zero magic).
emit_message($node, 0);
$prev_lsn = advance_out_of_record_splitting_zone($node);
$end_lsn = emit_message($node, 0);
$node->emit_wal(0);
$prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
$end_lsn = $node->emit_wal(0);
$node->stop('immediate');
write_wal($node, $TLI, $end_lsn,
$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
build_record_header(2 * 1024 * 1024 * 1024, 0, $prev_lsn));
$log_size = -s $node->logfile;
$node->start;
@ -347,16 +223,14 @@ ok($node->log_contains("invalid magic number 0000 .* LSN .*", $log_size),
"xlp_magic zero");
# Good xl_prev, we hit garbage page next (bad magic).
emit_message($node, 0);
$prev_lsn = advance_out_of_record_splitting_zone($node);
$end_lsn = emit_message($node, 0);
$node->emit_wal(0);
$prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
$end_lsn = $node->emit_wal(0);
$node->stop('immediate');
write_wal($node, $TLI, $end_lsn,
$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
build_record_header(2 * 1024 * 1024 * 1024, 0, $prev_lsn));
write_wal(
$node, $TLI,
start_of_next_page($end_lsn),
build_page_header(0xcafe, 0, 1, 0));
$node->write_wal($TLI, start_of_next_page($end_lsn),
$WAL_SEGMENT_SIZE, build_page_header(0xcafe, 0, 1, 0));
$log_size = -s $node->logfile;
$node->start;
ok($node->log_contains("invalid magic number CAFE .* LSN .*", $log_size),
@ -364,16 +238,14 @@ ok($node->log_contains("invalid magic number CAFE .* LSN .*", $log_size),
# Good xl_prev, we hit typical recycled page (good xlp_magic, bad
# xlp_pageaddr).
emit_message($node, 0);
$prev_lsn = advance_out_of_record_splitting_zone($node);
$end_lsn = emit_message($node, 0);
$node->emit_wal(0);
$prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
$end_lsn = $node->emit_wal(0);
$node->stop('immediate');
write_wal($node, $TLI, $end_lsn,
$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
build_record_header(2 * 1024 * 1024 * 1024, 0, $prev_lsn));
write_wal(
$node, $TLI,
start_of_next_page($end_lsn),
build_page_header($XLP_PAGE_MAGIC, 0, 1, 0xbaaaaaad));
$node->write_wal($TLI, start_of_next_page($end_lsn),
$WAL_SEGMENT_SIZE, build_page_header($XLP_PAGE_MAGIC, 0, 1, 0xbaaaaaad));
$log_size = -s $node->logfile;
$node->start;
ok( $node->log_contains(
@ -381,15 +253,16 @@ ok( $node->log_contains(
"xlp_pageaddr bad");
# Good xl_prev, xlp_magic, xlp_pageaddr, but bogus xlp_info.
emit_message($node, 0);
$prev_lsn = advance_out_of_record_splitting_zone($node);
$end_lsn = emit_message($node, 0);
$node->emit_wal(0);
$prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
$end_lsn = $node->emit_wal(0);
$node->stop('immediate');
write_wal($node, $TLI, $end_lsn,
$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
build_record_header(2 * 1024 * 1024 * 1024, 42, $prev_lsn));
write_wal(
$node, $TLI,
$node->write_wal(
$TLI,
start_of_next_page($end_lsn),
$WAL_SEGMENT_SIZE,
build_page_header(
$XLP_PAGE_MAGIC, 0x1234, 1, start_of_next_page($end_lsn)));
$log_size = -s $node->logfile;
@ -399,15 +272,14 @@ ok($node->log_contains("invalid info bits 1234 in .*, LSN .*,", $log_size),
# Good xl_prev, xlp_magic, xlp_pageaddr, but xlp_info doesn't mention
# continuation record.
emit_message($node, 0);
$prev_lsn = advance_out_of_record_splitting_zone($node);
$end_lsn = emit_message($node, 0);
$node->emit_wal(0);
$prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
$end_lsn = $node->emit_wal(0);
$node->stop('immediate');
write_wal($node, $TLI, $end_lsn,
$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
build_record_header(2 * 1024 * 1024 * 1024, 42, $prev_lsn));
write_wal(
$node, $TLI,
start_of_next_page($end_lsn),
$node->write_wal($TLI, start_of_next_page($end_lsn),
$WAL_SEGMENT_SIZE,
build_page_header($XLP_PAGE_MAGIC, 0, 1, start_of_next_page($end_lsn)));
$log_size = -s $node->logfile;
$node->start;
@ -416,15 +288,16 @@ ok($node->log_contains("there is no contrecord flag at .*", $log_size),
# Good xl_prev, xlp_magic, xlp_pageaddr, xlp_info but xlp_rem_len doesn't add
# up.
emit_message($node, 0);
$prev_lsn = advance_out_of_record_splitting_zone($node);
$end_lsn = emit_message($node, 0);
$node->emit_wal(0);
$prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
$end_lsn = $node->emit_wal(0);
$node->stop('immediate');
write_wal($node, $TLI, $end_lsn,
$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
build_record_header(2 * 1024 * 1024 * 1024, 42, $prev_lsn));
write_wal(
$node, $TLI,
$node->write_wal(
$TLI,
start_of_next_page($end_lsn),
$WAL_SEGMENT_SIZE,
build_page_header(
$XLP_PAGE_MAGIC, $XLP_FIRST_IS_CONTRECORD,
1, start_of_next_page($end_lsn),
@ -441,10 +314,10 @@ note "Multi-page, but header is split, so page checks are done first";
###########################################################################
# xl_prev is bad and xl_tot_len is too big, but we'll check xlp_magic first.
emit_message($node, 0);
$end_lsn = advance_to_record_splitting_zone($node);
$node->emit_wal(0);
$end_lsn = $node->advance_wal_to_record_splitting_zone($WAL_BLOCK_SIZE);
$node->stop('immediate');
write_wal($node, $TLI, $end_lsn,
$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef));
$log_size = -s $node->logfile;
$node->start;
@ -452,14 +325,15 @@ ok($node->log_contains("invalid magic number 0000 .* LSN .*", $log_size),
"xlp_magic zero (split record header)");
# And we'll also check xlp_pageaddr before any header checks.
emit_message($node, 0);
$end_lsn = advance_to_record_splitting_zone($node);
$node->emit_wal(0);
$end_lsn = $node->advance_wal_to_record_splitting_zone($WAL_BLOCK_SIZE);
$node->stop('immediate');
write_wal($node, $TLI, $end_lsn,
$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef));
write_wal(
$node, $TLI,
$node->write_wal(
$TLI,
start_of_next_page($end_lsn),
$WAL_SEGMENT_SIZE,
build_page_header(
$XLP_PAGE_MAGIC, $XLP_FIRST_IS_CONTRECORD, 1, 0xbaaaaaad));
$log_size = -s $node->logfile;
@ -470,14 +344,15 @@ ok( $node->log_contains(
# We'll also discover that xlp_rem_len doesn't add up before any
# header checks,
emit_message($node, 0);
$end_lsn = advance_to_record_splitting_zone($node);
$node->emit_wal(0);
$end_lsn = $node->advance_wal_to_record_splitting_zone($WAL_BLOCK_SIZE);
$node->stop('immediate');
write_wal($node, $TLI, $end_lsn,
$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef));
write_wal(
$node, $TLI,
$node->write_wal(
$TLI,
start_of_next_page($end_lsn),
$WAL_SEGMENT_SIZE,
build_page_header(
$XLP_PAGE_MAGIC, $XLP_FIRST_IS_CONTRECORD,
1, start_of_next_page($end_lsn),

Write Preview
Loading…
Cancel
Save