mirror of https://github.com/postgres/postgres
This feature allows skipping the transaction on subscriber nodes. If incoming change violates any constraint, logical replication stops until it's resolved. Currently, users need to either manually resolve the conflict by updating a subscriber-side database or by using function pg_replication_origin_advance() to skip the conflicting transaction. This commit introduces a simpler way to skip the conflicting transactions. The user can specify LSN by ALTER SUBSCRIPTION ... SKIP (lsn = XXX), which allows the apply worker to skip the transaction finished at specified LSN. The apply worker skips all data modification changes within the transaction. Author: Masahiko Sawada Reviewed-by: Takamichi Osumi, Hou Zhijie, Peter Eisentraut, Amit Kapila, Shi Yu, Vignesh C, Greg Nancarrow, Haiying Tang, Euler Taveira Discussion: https://postgr.es/m/CAD21AoDeScrsHhLyEPYqN3sydg6PxAPVBboK=30xJfUVihNZDA@mail.gmail.compull/81/head
Amit Kapila
4 years ago
parent
315ae75e9b
commit
208c5d65bb
@ -1,94 +0,0 @@ |
||||
|
||||
# Copyright (c) 2021-2022, PostgreSQL Global Development Group |
||||
|
||||
# Test of logical replication subscription self-disabling feature. |
||||
use strict; |
||||
use warnings; |
||||
use PostgreSQL::Test::Cluster; |
||||
use PostgreSQL::Test::Utils; |
||||
use Test::More; |
||||
|
||||
# create publisher node |
||||
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); |
||||
$node_publisher->init(allows_streaming => 'logical'); |
||||
$node_publisher->start; |
||||
|
||||
# create subscriber node |
||||
my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); |
||||
$node_subscriber->init; |
||||
$node_subscriber->start; |
||||
|
||||
# Create identical table on both nodes. |
||||
$node_publisher->safe_psql('postgres', "CREATE TABLE tbl (i INT)"); |
||||
$node_subscriber->safe_psql('postgres', "CREATE TABLE tbl (i INT)"); |
||||
|
||||
# Insert duplicate values on the publisher. |
||||
$node_publisher->safe_psql('postgres', |
||||
"INSERT INTO tbl (i) VALUES (1), (1), (1)"); |
||||
|
||||
# Create an additional unique index on the subscriber. |
||||
$node_subscriber->safe_psql('postgres', |
||||
"CREATE UNIQUE INDEX tbl_unique ON tbl (i)"); |
||||
|
||||
# Create a pub/sub to set up logical replication. This tests that the |
||||
# uniqueness violation will cause the subscription to fail during initial |
||||
# synchronization and make it disabled. |
||||
my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; |
||||
$node_publisher->safe_psql('postgres', |
||||
"CREATE PUBLICATION pub FOR TABLE tbl"); |
||||
$node_subscriber->safe_psql('postgres', |
||||
"CREATE SUBSCRIPTION sub CONNECTION '$publisher_connstr' PUBLICATION pub WITH (disable_on_error = true)" |
||||
); |
||||
|
||||
# Initial synchronization failure causes the subscription to be disabled. |
||||
$node_subscriber->poll_query_until('postgres', |
||||
"SELECT subenabled = false FROM pg_catalog.pg_subscription WHERE subname = 'sub'" |
||||
) or die "Timed out while waiting for subscriber to be disabled"; |
||||
|
||||
# Drop the unique index on the subscriber which caused the subscription to be |
||||
# disabled. |
||||
$node_subscriber->safe_psql('postgres', "DROP INDEX tbl_unique"); |
||||
|
||||
# Re-enable the subscription "sub". |
||||
$node_subscriber->safe_psql('postgres', "ALTER SUBSCRIPTION sub ENABLE"); |
||||
|
||||
# Wait for the data to replicate. |
||||
$node_publisher->wait_for_catchup('sub'); |
||||
$node_subscriber->poll_query_until('postgres', |
||||
"SELECT COUNT(1) = 0 FROM pg_subscription_rel sr WHERE sr.srsubstate NOT IN ('s', 'r') AND sr.srrelid = 'tbl'::regclass" |
||||
); |
||||
|
||||
# Confirm that we have finished the table sync. |
||||
my $result = |
||||
$node_subscriber->safe_psql('postgres', "SELECT MAX(i), COUNT(*) FROM tbl"); |
||||
is($result, qq(1|3), "subscription sub replicated data"); |
||||
|
||||
# Delete the data from the subscriber and recreate the unique index. |
||||
$node_subscriber->safe_psql('postgres', "DELETE FROM tbl"); |
||||
$node_subscriber->safe_psql('postgres', |
||||
"CREATE UNIQUE INDEX tbl_unique ON tbl (i)"); |
||||
|
||||
# Add more non-unique data to the publisher. |
||||
$node_publisher->safe_psql('postgres', |
||||
"INSERT INTO tbl (i) VALUES (3), (3), (3)"); |
||||
|
||||
# Apply failure causes the subscription to be disabled. |
||||
$node_subscriber->poll_query_until('postgres', |
||||
"SELECT subenabled = false FROM pg_catalog.pg_subscription WHERE subname = 'sub'" |
||||
) or die "Timed out while waiting for subscription sub to be disabled"; |
||||
|
||||
# Drop the unique index on the subscriber and re-enabled the subscription. Then |
||||
# confirm that the previously failing insert was applied OK. |
||||
$node_subscriber->safe_psql('postgres', "DROP INDEX tbl_unique"); |
||||
$node_subscriber->safe_psql('postgres', "ALTER SUBSCRIPTION sub ENABLE"); |
||||
|
||||
$node_publisher->wait_for_catchup('sub'); |
||||
|
||||
$result = $node_subscriber->safe_psql('postgres', |
||||
"SELECT COUNT(*) FROM tbl WHERE i = 3"); |
||||
is($result, qq(3), 'check the result of apply'); |
||||
|
||||
$node_subscriber->stop; |
||||
$node_publisher->stop; |
||||
|
||||
done_testing(); |
||||
@ -0,0 +1,183 @@ |
||||
|
||||
# Copyright (c) 2021-2022, PostgreSQL Global Development Group |
||||
|
||||
# Tests for disable_on_error and SKIP transaction features. |
||||
use strict; |
||||
use warnings; |
||||
use PostgreSQL::Test::Cluster; |
||||
use PostgreSQL::Test::Utils; |
||||
use Test::More; |
||||
|
||||
my $offset = 0; |
||||
|
||||
# Test skipping the transaction. This function must be called after the caller |
||||
# has inserted data that conflicts with the subscriber. The finish LSN of the |
||||
# error transaction that is used to specify to ALTER SUBSCRIPTION ... SKIP is |
||||
# fetched from the server logs. After executing ALTER SUBSCRITPION ... SKIP, we |
||||
# check if logical replication can continue working by inserting $nonconflict_data |
||||
# on the publisher. |
||||
sub test_skip_lsn |
||||
{ |
||||
my ($node_publisher, $node_subscriber, $nonconflict_data, $expected, $msg) |
||||
= @_; |
||||
|
||||
# Wait until a conflict occurs on the subscriber. |
||||
$node_subscriber->poll_query_until('postgres', |
||||
"SELECT subenabled = FALSE FROM pg_subscription WHERE subname = 'sub'" |
||||
); |
||||
|
||||
# Get the finish LSN of the error transaction. |
||||
my $contents = slurp_file($node_subscriber->logfile, $offset); |
||||
$contents =~ |
||||
qr/processing remote data for replication origin \"pg_\d+\" during "INSERT" for replication target relation "public.tbl" in transaction \d+ finished at ([[:xdigit:]]+\/[[:xdigit:]]+)/ |
||||
or die "could not get error-LSN"; |
||||
my $lsn = $1; |
||||
|
||||
# Set skip lsn. |
||||
$node_subscriber->safe_psql('postgres', |
||||
"ALTER SUBSCRIPTION sub SKIP (lsn = '$lsn')"); |
||||
|
||||
# Re-enable the subscription. |
||||
$node_subscriber->safe_psql('postgres', "ALTER SUBSCRIPTION sub ENABLE"); |
||||
|
||||
# Wait for the failed transaction to be skipped |
||||
$node_subscriber->poll_query_until('postgres', |
||||
"SELECT subskiplsn = '0/0' FROM pg_subscription WHERE subname = 'sub'" |
||||
); |
||||
|
||||
# Check the log to ensure that the transaction is skipped, and advance the |
||||
# offset of the log file for the next test. |
||||
$offset = $node_subscriber->wait_for_log( |
||||
qr/LOG: done skipping logical replication transaction finished at $lsn/, |
||||
$offset); |
||||
|
||||
# Insert non-conflict data |
||||
$node_publisher->safe_psql('postgres', |
||||
"INSERT INTO tbl VALUES $nonconflict_data"); |
||||
|
||||
$node_publisher->wait_for_catchup('sub'); |
||||
|
||||
# Check replicated data |
||||
my $res = |
||||
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tbl"); |
||||
is($res, $expected, $msg); |
||||
} |
||||
|
||||
# Create publisher node. Set a low value of logical_decoding_work_mem to test |
||||
# streaming cases. |
||||
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); |
||||
$node_publisher->init(allows_streaming => 'logical'); |
||||
$node_publisher->append_conf( |
||||
'postgresql.conf', |
||||
qq[ |
||||
logical_decoding_work_mem = 64kB |
||||
max_prepared_transactions = 10 |
||||
]); |
||||
$node_publisher->start; |
||||
|
||||
# Create subscriber node |
||||
my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); |
||||
$node_subscriber->init; |
||||
$node_subscriber->append_conf( |
||||
'postgresql.conf', |
||||
qq[ |
||||
max_prepared_transactions = 10 |
||||
]); |
||||
$node_subscriber->start; |
||||
|
||||
# Initial table setup on both publisher and subscriber. On the subscriber, we |
||||
# create the same tables but with a primary key. Also, insert some data that |
||||
# will conflict with the data replicated from publisher later. |
||||
$node_publisher->safe_psql( |
||||
'postgres', |
||||
qq[ |
||||
CREATE TABLE tbl (i INT, t TEXT); |
||||
INSERT INTO tbl VALUES (1, NULL); |
||||
]); |
||||
$node_subscriber->safe_psql( |
||||
'postgres', |
||||
qq[ |
||||
CREATE TABLE tbl (i INT PRIMARY KEY, t TEXT); |
||||
INSERT INTO tbl VALUES (1, NULL); |
||||
]); |
||||
|
||||
# Create a pub/sub to set up logical replication. This tests that the |
||||
# uniqueness violation will cause the subscription to fail during initial |
||||
# synchronization and make it disabled. |
||||
my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; |
||||
$node_publisher->safe_psql('postgres', |
||||
"CREATE PUBLICATION pub FOR TABLE tbl"); |
||||
$node_subscriber->safe_psql('postgres', |
||||
"CREATE SUBSCRIPTION sub CONNECTION '$publisher_connstr' PUBLICATION pub WITH (disable_on_error = true, streaming = on, two_phase = on)" |
||||
); |
||||
|
||||
# Initial synchronization failure causes the subscription to be disabled. |
||||
$node_subscriber->poll_query_until('postgres', |
||||
"SELECT subenabled = false FROM pg_catalog.pg_subscription WHERE subname = 'sub'" |
||||
) or die "Timed out while waiting for subscriber to be disabled"; |
||||
|
||||
# Truncate the table on the subscriber which caused the subscription to be |
||||
# disabled. |
||||
$node_subscriber->safe_psql('postgres', "TRUNCATE tbl"); |
||||
|
||||
# Re-enable the subscription "sub". |
||||
$node_subscriber->safe_psql('postgres', "ALTER SUBSCRIPTION sub ENABLE"); |
||||
|
||||
# Wait for the data to replicate. |
||||
$node_publisher->wait_for_catchup('sub'); |
||||
$node_subscriber->poll_query_until('postgres', |
||||
"SELECT COUNT(1) = 0 FROM pg_subscription_rel sr WHERE sr.srsubstate NOT IN ('s', 'r') AND sr.srrelid = 'tbl'::regclass" |
||||
); |
||||
|
||||
# Confirm that we have finished the table sync. |
||||
my $result = |
||||
$node_subscriber->safe_psql('postgres', "SELECT COUNT(*) FROM tbl"); |
||||
is($result, qq(1), "subscription sub replicated data"); |
||||
|
||||
# Insert data to tbl, raising an error on the subscriber due to violation |
||||
# of the unique constraint on tbl. Then skip the transaction. |
||||
$node_publisher->safe_psql( |
||||
'postgres', |
||||
qq[ |
||||
BEGIN; |
||||
INSERT INTO tbl VALUES (1, NULL); |
||||
COMMIT; |
||||
]); |
||||
test_skip_lsn($node_publisher, $node_subscriber, |
||||
"(2, NULL)", "2", "test skipping transaction"); |
||||
|
||||
# Test for PREPARE and COMMIT PREPARED. Insert the same data to tbl and |
||||
# PREPARE the transaction, raising an error. Then skip the transaction. |
||||
$node_publisher->safe_psql( |
||||
'postgres', |
||||
qq[ |
||||
BEGIN; |
||||
INSERT INTO tbl VALUES (1, NULL); |
||||
PREPARE TRANSACTION 'gtx'; |
||||
COMMIT PREPARED 'gtx'; |
||||
]); |
||||
test_skip_lsn($node_publisher, $node_subscriber, |
||||
"(3, NULL)", "3", "test skipping prepare and commit prepared "); |
||||
|
||||
# Test for STREAM COMMIT. Insert enough rows to tbl to exceed the 64kB |
||||
# limit, also raising an error on the subscriber during applying spooled |
||||
# changes for the same reason. Then skip the transaction. |
||||
$node_publisher->safe_psql( |
||||
'postgres', |
||||
qq[ |
||||
BEGIN; |
||||
INSERT INTO tbl SELECT i, md5(i::text) FROM generate_series(1, 10000) s(i); |
||||
COMMIT; |
||||
]); |
||||
test_skip_lsn($node_publisher, $node_subscriber, "(4, md5(4::text))", |
||||
"4", "test skipping stream-commit"); |
||||
|
||||
$result = $node_subscriber->safe_psql('postgres', |
||||
"SELECT COUNT(*) FROM pg_prepared_xacts"); |
||||
is($result, "0", |
||||
"check all prepared transactions are resolved on the subscriber"); |
||||
|
||||
$node_subscriber->stop; |
||||
$node_publisher->stop; |
||||
|
||||
done_testing(); |
||||
Loading…
Reference in new issue