Add more tests for REINDEX DATABASE/SYSTEM with relfilenode changes

Adding such commands in the main regression test suite is not a good
approach performance-wise as it impacts all the objects in the
regression database, so this additional coverage is added in the TAP
tests of reindexdb where we already run a few REINDEX commands with
SYSTEM and DATABASE so there is no runtime difference for the test.
This additional coverage checks which relations are rewritten with
relfilenode changes, as of:
- a toast index in user table.
- a toast index in catalog table.
- a catalog index.
- a user index.

This test suite is something I have implemented for a separate patch
that reworks a bit the way we handle these two REINDEX behaviors, but it
has enough value in itself to be in a separate commit.  This also makes
easier to follow what actually changes once the REINDEX logic is
reworked (currently, DABATASE rewrites both catalog and user tables, and
SYSTEM works only on catalogs).

Discussion: https://postgr.es/m/YtOqA7ldcJQADEE8@paquier.xyz

src/bin/scripts/t/090_reindexdb.pl

@@ -25,11 +25,6 @@ my $tbspace_name = 'reindex_tbspace';
 $node->safe_psql('postgres',
 	"CREATE TABLESPACE $tbspace_name LOCATION '$tbspace_path';");
 
-$node->issues_sql_like(
-	[ 'reindexdb', 'postgres' ],
-	qr/statement: REINDEX DATABASE postgres;/,
-	'SQL REINDEX run');
-
 # Use text as data type to get a toast table.
 $node->safe_psql('postgres',
 	'CREATE TABLE test1 (a text); CREATE INDEX test1x ON test1 (a);');
@@ -41,6 +36,74 @@ my $toast_index = $node->safe_psql('postgres',
 	"SELECT indexrelid::regclass FROM pg_index WHERE indrelid = '$toast_table'::regclass;"
 );
 
+# Set of SQL queries to cross-check the state of relfilenodes across
+# REINDEX operations.  A set of relfilenodes is saved from the catalogs
+# and then compared with pg_class.
+$node->safe_psql('postgres',
+	'CREATE TABLE toast_relfilenodes (parent regclass, indname regclass, relfilenode oid);'
+);
+# Save the relfilenode of a set of toast indexes, one from the catalog
+# pg_constraint and one from the test table.
+my $fetch_toast_relfilenodes =
+  qq{SELECT b.oid::regclass, c.oid::regclass, c.relfilenode
+  FROM pg_class a
+    JOIN pg_class b ON (a.oid = b.reltoastrelid)
+    JOIN pg_index i on (a.oid = i.indrelid)
+    JOIN pg_class c on (i.indexrelid = c.oid)
+  WHERE b.oid IN ('pg_constraint'::regclass, 'test1'::regclass)};
+# Same for relfilenodes of normal indexes.  This saves the relfilenode
+# from an index of pg_constraint, and from the index of the test table.
+my $fetch_index_relfilenodes = qq{SELECT i.indrelid, a.oid, a.relfilenode
+  FROM pg_class a
+    JOIN pg_index i ON (i.indexrelid = a.oid)
+  WHERE a.relname IN ('pg_constraint_oid_index', 'test1x')};
+my $save_relfilenodes =
+	"INSERT INTO toast_relfilenodes $fetch_toast_relfilenodes;"
+  . "INSERT INTO toast_relfilenodes $fetch_index_relfilenodes;";
+
+# Query to compare a set of relfilenodes saved with the contents of pg_class.
+# Note that this does not join using OIDs, as CONCURRENTLY would change them
+# when reindexing.  A filter is applied on the toast index names, even if this
+# does not make a difference between the catalog and normal ones.  The ordering
+# based on the name is enough to ensure a fixed output, where the name of the
+# parent table is included to provide more context.
+my $compare_relfilenodes = qq(SELECT b.parent::regclass,
+  regexp_replace(b.indname::text, '(pg_toast.pg_toast_)\\d{4,5}(_index)', '\\1<oid>\\2'),
+  CASE WHEN a.relfilenode = b.relfilenode THEN 'relfilenode is unchanged'
+    ELSE 'relfilenode has changed' END
+  FROM toast_relfilenodes b
+    JOIN pg_class a ON b.indname::text = a.oid::regclass::text
+  ORDER BY b.parent::text, b.indname::text);
+
+# Save the set of relfilenodes and compare them.
+$node->safe_psql('postgres', $save_relfilenodes);
+$node->issues_sql_like(
+	[ 'reindexdb', 'postgres' ],
+	qr/statement: REINDEX DATABASE postgres;/,
+	'SQL REINDEX run');
+my $relnode_info = $node->safe_psql('postgres', $compare_relfilenodes);
+is( $relnode_info,
+	qq(pg_constraint|pg_constraint_oid_index|relfilenode has changed
+pg_constraint|pg_toast.pg_toast_<oid>_index|relfilenode has changed
+test1|pg_toast.pg_toast_<oid>_index|relfilenode has changed
+test1|test1x|relfilenode has changed),
+	'relfilenode change after REINDEX DATABASE');
+
+# Re-save and run the second one.
+$node->safe_psql('postgres',
+	"TRUNCATE toast_relfilenodes; $save_relfilenodes");
+$node->issues_sql_like(
+	[ 'reindexdb', '-s', 'postgres' ],
+	qr/statement: REINDEX SYSTEM postgres;/,
+	'reindex system tables');
+$relnode_info = $node->safe_psql('postgres', $compare_relfilenodes);
+is( $relnode_info,
+	qq(pg_constraint|pg_constraint_oid_index|relfilenode has changed
+pg_constraint|pg_toast.pg_toast_<oid>_index|relfilenode has changed
+test1|pg_toast.pg_toast_<oid>_index|relfilenode is unchanged
+test1|test1x|relfilenode is unchanged),
+	'relfilenode change after REINDEX SYSTEM');
+
 $node->issues_sql_like(
 	[ 'reindexdb', '-t', 'test1', 'postgres' ],
 	qr/statement: REINDEX TABLE public\.test1;/,
@@ -57,10 +120,6 @@ $node->issues_sql_like(
 	[ 'reindexdb', '-S', 'pg_catalog', 'postgres' ],
 	qr/statement: REINDEX SCHEMA pg_catalog;/,
 	'reindex specific schema');
-$node->issues_sql_like(
-	[ 'reindexdb', '-s', 'postgres' ],
-	qr/statement: REINDEX SYSTEM postgres;/,
-	'reindex system tables');
 $node->issues_sql_like(
 	[ 'reindexdb', '-v', '-t', 'test1', 'postgres' ],
 	qr/statement: REINDEX \(VERBOSE\) TABLE public\.test1;/,