diff --git a/.github/workflows/postgresql-current-make-debug.yml b/.github/workflows/postgresql-current-make-debug.yml index d932db0f3ee..d056f222bad 100644 --- a/.github/workflows/postgresql-current-make-debug.yml +++ b/.github/workflows/postgresql-current-make-debug.yml @@ -39,7 +39,6 @@ jobs: cd contrib/pg_tde git checkout main git pull - ./configure working-directory: src - name: Build postgres diff --git a/configure b/configure index 18edb2c999e..886059ff93a 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for PostgreSQL 17.2. +# Generated by GNU Autoconf 2.69 for PostgreSQL 17.2.1. # # Report bugs to . # @@ -582,8 +582,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='PostgreSQL' PACKAGE_TARNAME='postgresql' -PACKAGE_VERSION='17.2' -PACKAGE_STRING='PostgreSQL 17.2' +PACKAGE_VERSION='17.2.1' +PACKAGE_STRING='PostgreSQL 17.2.1' PACKAGE_BUGREPORT='pgsql-bugs@lists.postgresql.org' PACKAGE_URL='https://www.postgresql.org/' @@ -658,6 +658,7 @@ UUID_LIBS LDAP_LIBS_BE LDAP_LIBS_FE with_ssl +enable_percona_ext PTHREAD_CFLAGS PTHREAD_LIBS PTHREAD_CC @@ -762,7 +763,6 @@ CPPFLAGS LDFLAGS CFLAGS CC -enable_percona_ext enable_injection_points enable_tap_tests enable_dtrace @@ -845,7 +845,6 @@ enable_coverage enable_dtrace enable_tap_tests enable_injection_points -enable_percona_ext with_blocksize with_segsize with_segsize_blocks @@ -861,6 +860,7 @@ with_python with_gssapi with_krb_srvnam with_pam +with_curl with_bsd_auth with_ldap with_bonjour @@ -878,6 +878,7 @@ with_lz4 with_zstd with_ssl with_openssl +enable_percona_ext enable_largefile ' ac_precious_vars='build_alias @@ -1540,9 +1541,9 @@ Optional Features: --enable-tap-tests enable TAP tests (requires Perl and IPC::Run) --enable-injection-points enable injection points (for testing) - --disable-percona-ext enable Percona specific features --enable-depend turn on automatic dependency tracking --enable-cassert enable assertion checks (for debugging) + --disable-percona-ext enable Percona specific features --disable-largefile omit support for large files Optional Packages: @@ -1572,6 +1573,7 @@ Optional Packages: --with-krb-srvnam=NAME default service principal name in Kerberos (GSSAPI) [postgres] --with-pam build with PAM support + --with-curl build with curl support --with-bsd-auth build with BSD Authentication support --with-ldap build with LDAP support --with-bonjour build with Bonjour support @@ -2835,7 +2837,7 @@ _ACEOF PG_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\)'` -PG_MINORVERSION=`expr "$PACKAGE_VERSION" : '.*\.\([0-9][0-9]*\)'` +PG_MINORVERSION=`expr "$PACKAGE_VERSION" : '.*\.\([0-9][0-9]*\)*\.'` test -n "$PG_MINORVERSION" || PG_MINORVERSION=0 @@ -2854,7 +2856,7 @@ cat >>confdefs.h <<_ACEOF _ACEOF -PG_PERCONAVERSION=1 +PG_PERCONAVERSION=`expr "$PACKAGE_VERSION" : '.*\.\([0-9][0-9]*\)'` cat >>confdefs.h <<_ACEOF #define PG_PERCONAVERSION "$PG_PERCONAVERSION" @@ -3726,38 +3728,6 @@ fi -# -# Percona ext -# - - -# Check whether --enable-percona-ext was given. -if test "${enable_percona_ext+set}" = set; then : - enableval=$enable_percona_ext; - case $enableval in - yes) - -$as_echo "#define PERCONA_EXT 1" >>confdefs.h - - ;; - no) - : - ;; - *) - as_fn_error $? "no argument expected for --enable-percona-ext option" "$LINENO" 5 - ;; - esac - -else - enable_percona_ext=yes - -$as_echo "#define PERCONA_EXT 1" >>confdefs.h - -fi - - - - # # Block size # @@ -8459,6 +8429,41 @@ fi $as_echo "$with_pam" >&6; } +# +# curl +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with curl support" >&5 +$as_echo_n "checking whether to build with curl support... " >&6; } + + + +# Check whether --with-curl was given. +if test "${with_curl+set}" = set; then : + withval=$with_curl; + case $withval in + yes) + +$as_echo "#define USE_CURL 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-curl option" "$LINENO" 5 + ;; + esac + +else + with_curl=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_curl" >&5 +$as_echo "$with_curl" >&6; } + + # # BSD AUTH # @@ -12371,6 +12376,43 @@ if test "$with_openssl" = yes ; then with_ssl=openssl fi +# +# Percona ext +# +# Requires Open SSL and curl +# + + +# Check whether --enable-percona-ext was given. +if test "${enable_percona_ext+set}" = set; then : + enableval=$enable_percona_ext; + case $enableval in + yes) + +$as_echo "#define PERCONA_EXT 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --enable-percona-ext option" "$LINENO" 5 + ;; + esac + +else + enable_percona_ext=yes + +$as_echo "#define PERCONA_EXT 1" >>confdefs.h + +fi + + + +with_curl=yes +with_ssl=openssl + + if test "$with_ssl" = openssl ; then # Minimum required OpenSSL version is 1.0.2 @@ -12653,6 +12695,56 @@ elif test "$with_ssl" != no ; then fi +if test "$with_curl" = "yes" ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for curl_easy_setopt in -lcurl" >&5 +$as_echo_n "checking for curl_easy_setopt in -lcurl... " >&6; } +if ${ac_cv_lib_curl_curl_easy_setopt+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lcurl $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char curl_easy_setopt (); +int +main () +{ +return curl_easy_setopt (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_curl_curl_easy_setopt=yes +else + ac_cv_lib_curl_curl_easy_setopt=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_easy_setopt" >&5 +$as_echo "$ac_cv_lib_curl_curl_easy_setopt" >&6; } +if test "x$ac_cv_lib_curl_curl_easy_setopt" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBCURL 1 +_ACEOF + + LIBS="-lcurl $LIBS" + +else + as_fn_error $? "library 'curl' is required for curl support" "$LINENO" 5 +fi + +fi + if test "$with_pam" = yes ; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking for pam_start in -lpam" >&5 $as_echo_n "checking for pam_start in -lpam... " >&6; } @@ -13823,6 +13915,19 @@ else fi +fi + +if test "$with_curl" = "yes" ; then + ac_fn_c_check_header_mongrel "$LINENO" "curl/curl.h" "ac_cv_header_curl_curl_h" "$ac_includes_default" +if test "x$ac_cv_header_curl_curl_h" = xyes; then : + +else + + as_fn_error $? "header file is required for curl support" "$LINENO" 5 + +fi + + fi if test "$with_pam" = yes ; then @@ -14941,7 +15046,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) +#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -14987,7 +15092,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) +#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15011,7 +15116,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) +#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15056,7 +15161,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) +#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15080,7 +15185,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) +#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; diff --git a/configure.ac b/configure.ac index bf0c37a271d..3e384bb91b6 100644 --- a/configure.ac +++ b/configure.ac @@ -17,7 +17,7 @@ dnl Read the Autoconf manual for details. dnl m4_pattern_forbid(^PGAC_)dnl to catch undefined macros -AC_INIT([PostgreSQL], [17.2], [pgsql-bugs@lists.postgresql.org], [], [https://www.postgresql.org/]) +AC_INIT([PostgreSQL], [17.2.1], [pgsql-bugs@lists.postgresql.org], [], [https://www.postgresql.org/]) m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.69 is required. Untested combinations of 'autoconf' and PostgreSQL versions are not @@ -30,14 +30,14 @@ AC_PREFIX_DEFAULT(/usr/local/pgsql) AC_DEFINE_UNQUOTED(CONFIGURE_ARGS, ["$ac_configure_args"], [Saved arguments from configure]) [PG_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\)'`] -[PG_MINORVERSION=`expr "$PACKAGE_VERSION" : '.*\.\([0-9][0-9]*\)'`] +[PG_MINORVERSION=`expr "$PACKAGE_VERSION" : '.*\.\([0-9][0-9]*\)*\.'`] test -n "$PG_MINORVERSION" || PG_MINORVERSION=0 AC_SUBST(PG_MAJORVERSION) AC_DEFINE_UNQUOTED(PG_MAJORVERSION, "$PG_MAJORVERSION", [PostgreSQL major version as a string]) AC_DEFINE_UNQUOTED(PG_MAJORVERSION_NUM, $PG_MAJORVERSION, [PostgreSQL major version number]) AC_DEFINE_UNQUOTED(PG_MINORVERSION_NUM, $PG_MINORVERSION, [PostgreSQL minor version number]) -[PG_PERCONAVERSION=1] +[PG_PERCONAVERSION=`expr "$PACKAGE_VERSION" : '.*\.\([0-9][0-9]*\)'`] AC_DEFINE_UNQUOTED(PG_PERCONAVERSION, "$PG_PERCONAVERSION", [PostgreSQL Percona version as a string]) PGAC_ARG_REQ(with, extra-version, [STRING], [append STRING to version], @@ -259,13 +259,6 @@ PGAC_ARG_BOOL(enable, injection-points, no, [enable injection points (for testin [AC_DEFINE([USE_INJECTION_POINTS], 1, [Define to 1 to build with injection points. (--enable-injection-points)])]) AC_SUBST(enable_injection_points) -# -# Percona ext -# -PGAC_ARG_BOOL(enable, percona-ext, yes, [enable Percona specific features], - [AC_DEFINE([PERCONA_EXT], 1, [Define to 1 to build with Percona specific features. (--enable-percona-ext)])]) -AC_SUBST(enable_percona_ext) - # # Block size # @@ -916,6 +909,16 @@ PGAC_ARG_BOOL(with, pam, no, AC_MSG_RESULT([$with_pam]) +# +# curl +# +AC_MSG_CHECKING([whether to build with curl support]) +PGAC_ARG_BOOL(with, curl, no, + [build with curl support], + [AC_DEFINE([USE_CURL], 1, [Define to 1 to build with curl support. (--with-curl)])]) +AC_MSG_RESULT([$with_curl]) + + # # BSD AUTH # @@ -1343,6 +1346,18 @@ if test "$with_openssl" = yes ; then with_ssl=openssl fi +# +# Percona ext +# +# Requires Open SSL and curl +# +PGAC_ARG_BOOL(enable, percona-ext, yes, [enable Percona specific features], + [AC_DEFINE([PERCONA_EXT], 1, [Define to 1 to build with Percona specific features. (--enable-percona-ext)])]) + +with_curl=yes +with_ssl=openssl +AC_SUBST(enable_percona_ext) + if test "$with_ssl" = openssl ; then dnl Order matters! # Minimum required OpenSSL version is 1.0.2 @@ -1375,6 +1390,10 @@ elif test "$with_ssl" != no ; then fi AC_SUBST(with_ssl) +if test "$with_curl" = "yes" ; then + AC_CHECK_LIB(curl, curl_easy_setopt, [], [AC_MSG_ERROR([library 'curl' is required for curl support])]) +fi + if test "$with_pam" = yes ; then AC_CHECK_LIB(pam, pam_start, [], [AC_MSG_ERROR([library 'pam' is required for PAM])]) fi @@ -1554,6 +1573,13 @@ if test "$with_ssl" = openssl ; then AC_CHECK_HEADER(openssl/err.h, [], [AC_MSG_ERROR([header file is required for OpenSSL])]) fi +if test "$with_curl" = "yes" ; then + AC_CHECK_HEADER(curl/curl.h, [], + [ + AC_MSG_ERROR([header file is required for curl support]) + ]) +fi + if test "$with_pam" = yes ; then AC_CHECK_HEADERS(security/pam_appl.h, [], [AC_CHECK_HEADERS(pam/pam_appl.h, [], diff --git a/contrib/Makefile b/contrib/Makefile index abd780f2774..272c59fb761 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -35,6 +35,7 @@ SUBDIRS = \ pg_prewarm \ pg_stat_statements \ pg_surgery \ + pg_tde \ pg_trgm \ pgrowlocks \ pgstattuple \ diff --git a/contrib/pg_tde b/contrib/pg_tde index ae29fd7e522..e0978a8be6c 160000 --- a/contrib/pg_tde +++ b/contrib/pg_tde @@ -1 +1 @@ -Subproject commit ae29fd7e522deb54b2aa405ac9babc53576ac617 +Subproject commit e0978a8be6c70b2fccc86ca1cb8fc5499dd83a88 diff --git a/meson.build b/meson.build index ff6768823d0..f2cb30c4f54 100644 --- a/meson.build +++ b/meson.build @@ -1322,6 +1322,11 @@ if sslopt == 'auto' and auto_features.disabled() sslopt = 'none' endif +# OpenSSL is required for Percona extentions +if percona_ext == true + sslopt = 'openssl' +endif + if sslopt in ['auto', 'openssl'] openssl_required = (sslopt == 'openssl') diff --git a/src/Makefile.global.in b/src/Makefile.global.in index a00c909681e..ca305b5664d 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -203,6 +203,7 @@ enable_dtrace = @enable_dtrace@ enable_coverage = @enable_coverage@ enable_injection_points = @enable_injection_points@ enable_tap_tests = @enable_tap_tests@ +enable_percona_ext = @enable_percona_ext@ python_includespec = @python_includespec@ python_libdir = @python_libdir@ diff --git a/src/bin/meson.build b/src/bin/meson.build index aa60ebaa302..093f049dd08 100644 --- a/src/bin/meson.build +++ b/src/bin/meson.build @@ -1,5 +1,44 @@ # Copyright (c) 2022-2024, PostgreSQL Global Development Group +tde_decrypt_sources = [] +tde_include = [] +tde_deps = [] + +if percona_ext == true +# TODO: should be in pg_tde, ideally as a static lib +tde_decrypt_sources = files( + '../../contrib/pg_tde/src/access/pg_tde_tdemap.c', + '../../contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c', + '../../contrib/pg_tde/src/catalog/tde_global_space.c', + '../../contrib/pg_tde/src/catalog/tde_keyring.c', + '../../contrib/pg_tde/src/catalog/tde_keyring_parse_opts.c', + '../../contrib/pg_tde/src/catalog/tde_principal_key.c', + '../../contrib/pg_tde/src/common/pg_tde_utils.c', + '../../contrib/pg_tde/src/encryption/enc_aes.c', + '../../contrib/pg_tde/src/encryption/enc_tde.c', + '../../contrib/pg_tde/src/keyring/keyring_api.c', + '../../contrib/pg_tde/src/keyring/keyring_curl.c', + '../../contrib/pg_tde/src/keyring/keyring_file.c', + '../../contrib/pg_tde/src/keyring/keyring_vault.c', + '../../contrib/pg_tde/src/keyring/keyring_kmip.c', + '../../contrib/pg_tde/src/keyring/keyring_kmip_ereport.c', + + '../../contrib/pg_tde/src/libkmip/libkmip/src/kmip.c', + '../../contrib/pg_tde/src/libkmip/libkmip/src/kmip_bio.c', + '../../contrib/pg_tde/src/libkmip/libkmip/src/kmip_locate.c', + '../../contrib/pg_tde/src/libkmip/libkmip/src/kmip_memset.c', + ) + +tde_include = include_directories( + '../../contrib/pg_tde/src/include', + '../../contrib/pg_tde/src/libkmip/libkmip/include' + ) + +curldep = dependency('libcurl') +tde_deps = [curldep] + +endif + subdir('initdb') subdir('pg_amcheck') subdir('pg_archivecleanup') diff --git a/src/bin/pg_waldump/Makefile b/src/bin/pg_waldump/Makefile index 4c1ee649501..b7307caaf7f 100644 --- a/src/bin/pg_waldump/Makefile +++ b/src/bin/pg_waldump/Makefile @@ -18,6 +18,19 @@ OBJS = \ override CPPFLAGS := -DFRONTEND $(CPPFLAGS) +ifeq ($(enable_percona_ext),yes) +# TODO: Ideally should be built in pg_tde as a static lib +include $(top_srcdir)/contrib/pg_tde/Makefile.tools + +TDE_OBJS2 = $(TDE_OBJS:%=$(top_srcdir)/contrib/pg_tde/%) + +OBJS += \ + $(top_srcdir)/src/fe_utils/simple_list.o \ + $(TDE_OBJS2) + +override CPPFLAGS := -I$(top_srcdir)/contrib/pg_tde/src/include -I$(top_srcdir)/contrib/pg_tde/src/libkmip/libkmip/include $(CPPFLAGS) +endif + RMGRDESCSOURCES = $(sort $(notdir $(wildcard $(top_srcdir)/src/backend/access/rmgrdesc/*desc*.c))) RMGRDESCOBJS = $(patsubst %.c,%.o,$(RMGRDESCSOURCES)) diff --git a/src/bin/pg_waldump/meson.build b/src/bin/pg_waldump/meson.build index bb30f0fe08e..24130f1f281 100644 --- a/src/bin/pg_waldump/meson.build +++ b/src/bin/pg_waldump/meson.build @@ -9,6 +9,7 @@ pg_waldump_sources = files( pg_waldump_sources += rmgr_desc_sources pg_waldump_sources += xlogreader_sources pg_waldump_sources += files('../../backend/access/transam/xlogstats.c') +pg_waldump_sources += tde_decrypt_sources if host_system == 'windows' pg_waldump_sources += rc_bin_gen.process(win32ver_rc, extra_args: [ @@ -18,9 +19,10 @@ endif pg_waldump = executable('pg_waldump', pg_waldump_sources, - dependencies: [frontend_code, lz4, zstd], + dependencies: [frontend_code, lz4, zstd, tde_deps], c_args: ['-DFRONTEND'], # needed for xlogreader et al kwargs: default_bin_args, + include_directories: [postgres_inc, tde_include], ) bin_targets += pg_waldump @@ -32,6 +34,8 @@ tests += { 'tests': [ 't/001_basic.pl', 't/002_save_fullpage.pl', + 't/003_basic_encrypted.pl', + 't/004_save_fullpage_encrypted.pl', ], }, } diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 016825f46c7..d4a91eb76ad 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -32,6 +32,10 @@ #include "rmgrdesc.h" #include "storage/bufpage.h" +#ifdef PERCONA_EXT +#include "access/pg_tde_xlog_encrypt_fe.h" +#endif + /* * NOTE: For any code change or issue fix here, it is highly recommended to * give a thought about doing the same in pg_walinspect contrib module as well. @@ -760,30 +764,35 @@ usage(void) printf(_("Usage:\n")); printf(_(" %s [OPTION]... [STARTSEG [ENDSEG]]\n"), progname); printf(_("\nOptions:\n")); - printf(_(" -b, --bkp-details output detailed information about backup blocks\n")); - printf(_(" -B, --block=N with --relation, only show records that modify block N\n")); - printf(_(" -e, --end=RECPTR stop reading at WAL location RECPTR\n")); - printf(_(" -f, --follow keep retrying after reaching end of WAL\n")); - printf(_(" -F, --fork=FORK only show records that modify blocks in fork FORK;\n" - " valid names are main, fsm, vm, init\n")); - printf(_(" -n, --limit=N number of records to display\n")); - printf(_(" -p, --path=PATH directory in which to find WAL segment files or a\n" - " directory with a ./pg_wal that contains such files\n" - " (default: current directory, ./pg_wal, $PGDATA/pg_wal)\n")); - printf(_(" -q, --quiet do not print any output, except for errors\n")); - printf(_(" -r, --rmgr=RMGR only show records generated by resource manager RMGR;\n" - " use --rmgr=list to list valid resource manager names\n")); - printf(_(" -R, --relation=T/D/R only show records that modify blocks in relation T/D/R\n")); - printf(_(" -s, --start=RECPTR start reading at WAL location RECPTR\n")); - printf(_(" -t, --timeline=TLI timeline from which to read WAL records\n" - " (default: 1 or the value used in STARTSEG)\n")); - printf(_(" -V, --version output version information, then exit\n")); - printf(_(" -w, --fullpage only show records with a full page write\n")); - printf(_(" -x, --xid=XID only show records with transaction ID XID\n")); - printf(_(" -z, --stats[=record] show statistics instead of records\n" - " (optionally, show per-record statistics)\n")); - printf(_(" --save-fullpage=DIR save full page images to DIR\n")); - printf(_(" -?, --help show this help, then exit\n")); + printf(_(" -b, --bkp-details output detailed information about backup blocks\n")); + printf(_(" -B, --block=N with --relation, only show records that modify block N\n")); + printf(_(" -e, --end=RECPTR stop reading at WAL location RECPTR\n")); + printf(_(" -f, --follow keep retrying after reaching end of WAL\n")); + printf(_(" -F, --fork=FORK only show records that modify blocks in fork FORK;\n" + " valid names are main, fsm, vm, init\n")); + printf(_(" -n, --limit=N number of records to display\n")); + printf(_(" -p, --path=PATH directory in which to find WAL segment files or a\n" + " directory with a ./pg_wal that contains such files\n" + " (default: current directory, ./pg_wal, $PGDATA/pg_wal)\n")); +#ifdef PERCONA_EXT + printf(_(" -k, --keyring-path=PATH directory in which to find keyring config files for WAL\n" + " such files are pg_tde.map, pg_tde.dat, and pg_tde_keyrings\n" + " (it will not try to decrypt WAL if not set)\n")); +#endif + printf(_(" -q, --quiet do not print any output, except for errors\n")); + printf(_(" -r, --rmgr=RMGR only show records generated by resource manager RMGR;\n" + " use --rmgr=list to list valid resource manager names\n")); + printf(_(" -R, --relation=T/D/R only show records that modify blocks in relation T/D/R\n")); + printf(_(" -s, --start=RECPTR start reading at WAL location RECPTR\n")); + printf(_(" -t, --timeline=TLI timeline from which to read WAL records\n" + " (default: 1 or the value used in STARTSEG)\n")); + printf(_(" -V, --version output version information, then exit\n")); + printf(_(" -w, --fullpage only show records with a full page write\n")); + printf(_(" -x, --xid=XID only show records with transaction ID XID\n")); + printf(_(" -z, --stats[=record] show statistics instead of records\n" + " (optionally, show per-record statistics)\n")); + printf(_(" --save-fullpage=DIR save full page images to DIR\n")); + printf(_(" -?, --help show this help, then exit\n")); printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT); printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL); } @@ -800,6 +809,9 @@ main(int argc, char **argv) XLogRecord *record; XLogRecPtr first_record; char *waldir = NULL; +#ifdef PERCONA_EXT + char *kringdir = NULL; +#endif char *errormsg; static struct option long_options[] = { @@ -812,6 +824,9 @@ main(int argc, char **argv) {"help", no_argument, NULL, '?'}, {"limit", required_argument, NULL, 'n'}, {"path", required_argument, NULL, 'p'}, +#ifdef PERCONA_EXT + {"keyring-path", optional_argument, NULL, 'k'}, +#endif {"quiet", no_argument, NULL, 'q'}, {"relation", required_argument, NULL, 'R'}, {"rmgr", required_argument, NULL, 'r'}, @@ -885,7 +900,7 @@ main(int argc, char **argv) goto bad_argument; } - while ((option = getopt_long(argc, argv, "bB:e:fF:n:p:qr:R:s:t:wx:z", + while ((option = getopt_long(argc, argv, "bB:e:fF:n:p:k:qr:R:s:t:wx:z", long_options, &optindex)) != -1) { switch (option) @@ -934,6 +949,11 @@ main(int argc, char **argv) case 'p': waldir = pg_strdup(optarg); break; +#ifdef PERCONA_EXT + case 'k': + kringdir = pg_strdup(optarg); + break; +#endif case 'q': config.quiet = true; break; @@ -1106,6 +1126,16 @@ main(int argc, char **argv) } } +#ifdef PERCONA_EXT + /* + * Make possible to read ecrypted WAL + */ + if (kringdir != NULL) + { + TDE_XLOG_INIT(kringdir); + } +#endif + if (config.save_fullpage_path != NULL) create_fullpage_directory(config.save_fullpage_path); diff --git a/src/bin/pg_waldump/t/003_basic_encrypted.pl b/src/bin/pg_waldump/t/003_basic_encrypted.pl new file mode 100644 index 00000000000..81489e56bee --- /dev/null +++ b/src/bin/pg_waldump/t/003_basic_encrypted.pl @@ -0,0 +1,247 @@ + +# Copyright (c) 2021-2024, PostgreSQL Global Development Group + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf( + 'postgresql.conf', q{ +autovacuum = off +checkpoint_timeout = 1h + +# for standbydesc +archive_mode=on +archive_command='' + +# for XLOG_HEAP_TRUNCATE +wal_level=logical + +# WAL Encryption +shared_preload_libraries = 'pg_tde' +pg_tde.wal_encrypt = on +}); +$node->start; + +my ($start_lsn, $start_walfile) = split /\|/, + $node->safe_psql('postgres', + q{SELECT pg_current_wal_insert_lsn(), pg_walfile_name(pg_current_wal_insert_lsn())} + ); + +$node->safe_psql( + 'postgres', q{ +-- heap, btree, hash, sequence +CREATE TABLE t1 (a int GENERATED ALWAYS AS IDENTITY, b text); +CREATE INDEX i1a ON t1 USING btree (a); +CREATE INDEX i1b ON t1 USING hash (b); +INSERT INTO t1 VALUES (default, 'one'), (default, 'two'); +DELETE FROM t1 WHERE b = 'one'; +TRUNCATE t1; + +-- abort +START TRANSACTION; +INSERT INTO t1 VALUES (default, 'three'); +ROLLBACK; + +-- unlogged/init fork +CREATE UNLOGGED TABLE t2 (x int); +CREATE INDEX i2 ON t2 USING btree (x); +INSERT INTO t2 SELECT generate_series(1, 10); + +-- gin +CREATE TABLE gin_idx_tbl (id bigserial PRIMARY KEY, data jsonb); +CREATE INDEX gin_idx ON gin_idx_tbl USING gin (data); +INSERT INTO gin_idx_tbl + WITH random_json AS ( + SELECT json_object_agg(key, trunc(random() * 10)) as json_data + FROM unnest(array['a', 'b', 'c']) as u(key)) + SELECT generate_series(1,500), json_data FROM random_json; + +-- gist, spgist +CREATE TABLE gist_idx_tbl (p point); +CREATE INDEX gist_idx ON gist_idx_tbl USING gist (p); +CREATE INDEX spgist_idx ON gist_idx_tbl USING spgist (p); +INSERT INTO gist_idx_tbl (p) VALUES (point '(1, 1)'), (point '(3, 2)'), (point '(6, 3)'); + +-- brin +CREATE TABLE brin_idx_tbl (col1 int, col2 text, col3 text ); +CREATE INDEX brin_idx ON brin_idx_tbl USING brin (col1, col2, col3) WITH (autosummarize=on); +INSERT INTO brin_idx_tbl SELECT generate_series(1, 10000), 'dummy', 'dummy'; +UPDATE brin_idx_tbl SET col2 = 'updated' WHERE col1 BETWEEN 1 AND 5000; +SELECT brin_summarize_range('brin_idx', 0); +SELECT brin_desummarize_range('brin_idx', 0); + +VACUUM; + +-- logical message +SELECT pg_logical_emit_message(true, 'foo', 'bar'); + +-- relmap +VACUUM FULL pg_authid; + +-- database +CREATE DATABASE d1; +DROP DATABASE d1; +}); + +my $tblspc_path = PostgreSQL::Test::Utils::tempdir_short(); + +$node->safe_psql( + 'postgres', qq{ +CREATE TABLESPACE ts1 LOCATION '$tblspc_path'; +DROP TABLESPACE ts1; +}); + +my ($end_lsn, $end_walfile) = split /\|/, + $node->safe_psql('postgres', + q{SELECT pg_current_wal_insert_lsn(), pg_walfile_name(pg_current_wal_insert_lsn())} + ); + +my $default_ts_oid = $node->safe_psql('postgres', + q{SELECT oid FROM pg_tablespace WHERE spcname = 'pg_default'}); +my $postgres_db_oid = $node->safe_psql('postgres', + q{SELECT oid FROM pg_database WHERE datname = 'postgres'}); +my $rel_t1_oid = $node->safe_psql('postgres', + q{SELECT oid FROM pg_class WHERE relname = 't1'}); +my $rel_i1a_oid = $node->safe_psql('postgres', + q{SELECT oid FROM pg_class WHERE relname = 'i1a'}); + +$node->stop; + + +# various ways of specifying WAL range +command_fails_like( + [ 'pg_waldump', 'foo', 'bar' ], + qr/error: could not locate WAL file "foo"/, + 'start file not found'); +command_like([ 'pg_waldump', '-k', $node->data_dir. '/pg_tde', $node->data_dir . '/pg_wal/' . $start_walfile ], + qr/./, 'runs with start segment specified'); +command_fails_like( + [ 'pg_waldump', $node->data_dir . '/pg_wal/' . $start_walfile, 'bar' ], + qr/error: could not open file "bar"/, + 'end file not found'); +command_like( + [ + 'pg_waldump', + '-k', $node->data_dir. '/pg_tde', + $node->data_dir . '/pg_wal/' . $start_walfile, + $node->data_dir . '/pg_wal/' . $end_walfile + ], + qr/./, + 'runs with start and end segment specified'); +command_fails_like( + [ 'pg_waldump', '-p', $node->data_dir ], + qr/error: no start WAL location given/, + 'path option requires start location'); +command_like( + [ + 'pg_waldump', '-p', $node->data_dir, '--start', + $start_lsn, '--end', $end_lsn, + '-k', $node->data_dir. '/pg_tde' + ], + qr/./, + 'runs with path option and start and end locations'); +command_fails_like( + [ 'pg_waldump', '-k', $node->data_dir. '/pg_tde', '-p', $node->data_dir, '--start', $start_lsn ], + qr/error: error in WAL record at/, + 'falling off the end of the WAL results in an error'); + +command_like( + [ + 'pg_waldump', '--quiet', + '-k', $node->data_dir. '/pg_tde', + $node->data_dir . '/pg_wal/' . $start_walfile + ], + qr/^$/, + 'no output with --quiet option'); +command_fails_like( + [ 'pg_waldump', '--quiet', '-k', $node->data_dir. '/pg_tde', '-p', $node->data_dir, '--start', $start_lsn ], + qr/error: error in WAL record at/, + 'errors are shown with --quiet'); + + +# Test for: Display a message that we're skipping data if `from` +# wasn't a pointer to the start of a record. +{ + # Construct a new LSN that is one byte past the original + # start_lsn. + my ($part1, $part2) = split qr{/}, $start_lsn; + my $lsn2 = hex $part2; + $lsn2++; + my $new_start = sprintf("%s/%X", $part1, $lsn2); + + my (@cmd, $stdout, $stderr, $result); + + @cmd = ( + 'pg_waldump', '-k', $node->data_dir. '/pg_tde', + '--start', $new_start, + $node->data_dir . '/pg_wal/' . $start_walfile); + $result = IPC::Run::run \@cmd, '>', \$stdout, '2>', \$stderr; + ok($result, "runs with start segment and start LSN specified"); + like($stderr, qr/first record is after/, 'info message printed'); +} + + +# Helper function to test various options. Pass options as arguments. +# Output lines are returned as array. +sub test_pg_waldump +{ + local $Test::Builder::Level = $Test::Builder::Level + 1; + my @opts = @_; + + my (@cmd, $stdout, $stderr, $result, @lines); + + @cmd = ( + 'pg_waldump', '-k', $node->data_dir. '/pg_tde', '-p', $node->data_dir, + '--start', $start_lsn, '--end', $end_lsn); + push @cmd, @opts; + $result = IPC::Run::run \@cmd, '>', \$stdout, '2>', \$stderr; + ok($result, "pg_waldump @opts: runs ok"); + is($stderr, '', "pg_waldump @opts: no stderr"); + @lines = split /\n/, $stdout; + ok(@lines > 0, "pg_waldump @opts: some lines are output"); + return @lines; +} + +my @lines; + +@lines = test_pg_waldump; +is(grep(!/^rmgr: \w/, @lines), 0, 'all output lines are rmgr lines'); + +@lines = test_pg_waldump('--limit', 6); +is(@lines, 6, 'limit option observed'); + +@lines = test_pg_waldump('--fullpage'); +is(grep(!/^rmgr:.*\bFPW\b/, @lines), 0, 'all output lines are FPW'); + +@lines = test_pg_waldump('--stats'); +like($lines[0], qr/WAL statistics/, "statistics on stdout"); +is(grep(/^rmgr:/, @lines), 0, 'no rmgr lines output'); + +@lines = test_pg_waldump('--stats=record'); +like($lines[0], qr/WAL statistics/, "statistics on stdout"); +is(grep(/^rmgr:/, @lines), 0, 'no rmgr lines output'); + +@lines = test_pg_waldump('--rmgr', 'Btree'); +is(grep(!/^rmgr: Btree/, @lines), 0, 'only Btree lines'); + +@lines = test_pg_waldump('--fork', 'init'); +is(grep(!/fork init/, @lines), 0, 'only init fork lines'); + +@lines = test_pg_waldump('--relation', + "$default_ts_oid/$postgres_db_oid/$rel_t1_oid"); +is(grep(!/rel $default_ts_oid\/$postgres_db_oid\/$rel_t1_oid/, @lines), + 0, 'only lines for selected relation'); + +@lines = + test_pg_waldump('--relation', + "$default_ts_oid/$postgres_db_oid/$rel_i1a_oid", + '--block', 1); +is(grep(!/\bblk 1\b/, @lines), 0, 'only lines for selected block'); + + +done_testing(); \ No newline at end of file diff --git a/src/bin/pg_waldump/t/004_save_fullpage_encrypted.pl b/src/bin/pg_waldump/t/004_save_fullpage_encrypted.pl new file mode 100644 index 00000000000..8932f6a98de --- /dev/null +++ b/src/bin/pg_waldump/t/004_save_fullpage_encrypted.pl @@ -0,0 +1,117 @@ + +# Copyright (c) 2022-2024, PostgreSQL Global Development Group + +use strict; +use warnings FATAL => 'all'; +use File::Basename; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::RecursiveCopy; +use PostgreSQL::Test::Utils; +use Test::More; + +my ($blocksize, $walfile_name); + +# Function to extract the LSN from the given block structure +sub get_block_lsn +{ + my $path = shift; + my $blocksize = shift; + my $block; + + open my $fh, '<', $path or die "couldn't open file: $path\n"; + die "could not read block\n" + if $blocksize != read($fh, $block, $blocksize); + my ($lsn_hi, $lsn_lo) = unpack('LL', $block); + + $lsn_hi = sprintf('%08X', $lsn_hi); + $lsn_lo = sprintf('%08X', $lsn_lo); + + return ($lsn_hi, $lsn_lo); +} + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf( + 'postgresql.conf', q{ +wal_level = 'replica' +max_wal_senders = 4 + +shared_preload_libraries = 'pg_tde' +pg_tde.wal_encrypt = on +}); +$node->start; + +# Generate data/WAL to examine that will have full pages in them. +$node->safe_psql( + 'postgres', + "SELECT 'init' FROM pg_create_physical_replication_slot('regress_pg_waldump_slot', true, false); +CREATE TABLE test_table AS SELECT generate_series(1,100) a; +-- Force FPWs on the next writes. +CHECKPOINT; +UPDATE test_table SET a = a + 1; +"); + +($walfile_name, $blocksize) = split '\|' => $node->safe_psql('postgres', + "SELECT pg_walfile_name(pg_switch_wal()), current_setting('block_size')"); + +# Get the relation node, etc for the new table +my $relation = $node->safe_psql( + 'postgres', + q{SELECT format( + '%s/%s/%s', + CASE WHEN reltablespace = 0 THEN dattablespace ELSE reltablespace END, + pg_database.oid, + pg_relation_filenode(pg_class.oid)) + FROM pg_class, pg_database + WHERE relname = 'test_table' AND + datname = current_database()} +); + +my $walfile = $node->data_dir . '/pg_wal/' . $walfile_name; +my $tmp_folder = PostgreSQL::Test::Utils::tempdir; + +ok(-f $walfile, "Got a WAL file"); + +$node->command_ok( + [ + 'pg_waldump', '--quiet', + '-k', $node->data_dir. '/pg_tde', + '--save-fullpage', "$tmp_folder/raw", + '--relation', $relation, + $walfile + ], + 'pg_waldump with --save-fullpage runs'); + +# This regexp will match filenames formatted as: +# TLI-LSNh-LSNl.TBLSPCOID.DBOID.NODEOID.dd_fork with the components being: +# - Timeline ID in hex format. +# - WAL LSN in hex format, as two 8-character numbers. +# - Tablespace OID (0 for global). +# - Database OID. +# - Relfilenode. +# - Block number. +# - Fork this block came from (vm, init, fsm, or main). +my $file_re = + qr/^[0-9A-F]{8}-([0-9A-F]{8})-([0-9A-F]{8})[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+(?:_vm|_init|_fsm|_main)?$/; + +my $file_count = 0; + +# Verify filename format matches --save-fullpage. +for my $fullpath (glob "$tmp_folder/raw/*") +{ + my $file = File::Basename::basename($fullpath); + + like($file, $file_re, "verify filename format for file $file"); + $file_count++; + + my ($hi_lsn_fn, $lo_lsn_fn) = ($file =~ $file_re); + my ($hi_lsn_bk, $lo_lsn_bk) = get_block_lsn($fullpath, $blocksize); + + # The LSN on the block comes before the file's LSN. + ok( $hi_lsn_fn . $lo_lsn_fn gt $hi_lsn_bk . $lo_lsn_bk, + 'LSN stored in the file precedes the one stored in the block'); +} + +ok($file_count > 0, 'verify that at least one block has been saved'); + +done_testing(); \ No newline at end of file