Remove maketools and fix frontend tools tde data usage (#362)

This commit:
1. Removes autoconf builds for pg_tde so it can be together built with Postgres (now, used would have to go to contrib/pg_tde and build it explicitly after building Postgres) with make. There are still left pg_tde builds in CI tests since this PR depends on https://github.com/percona/postgres/pull/20. So those leftover will be removed after merging PG PR
2. Adds necessary changes regarding new code (like kmip) so frontend tools (pg_waldump et al) can be compiled with pg_tde
2. Get rid of realpath as it has issues with optimised builds

For: PG-1003, PG-1005
pull/209/head
Andrew Pogrebnoi 10 months ago committed by GitHub
parent 86f7995e5a
commit e0978a8be6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 4
      .github/workflows/check.yaml
  2. 1
      .github/workflows/postgresql-16-ppg-package-pgxs.yml
  3. 1
      .github/workflows/postgresql-16-src-make-ssl11.yml
  4. 1
      .github/workflows/postgresql-16-src-make.yml
  5. 1
      .github/workflows/postgresql-17-src-make.yml
  6. 1
      .github/workflows/postgresql-pgdg-package-pgxs.yml
  7. 2
      .gitignore
  8. 4
      Makefile
  9. 20
      Makefile.tools
  10. 1
      README.md
  11. 1807
      config.guess
  12. 1960
      config.sub
  13. 4984
      configure
  14. 63
      configure.ac
  15. 3
      docker/Dockerfile
  16. 1
      documentation/docs/contribute.md
  17. 1
      documentation/docs/install.md
  18. 12
      meson.build
  19. 2
      pgindent_excludes
  20. 9
      src/catalog/tde_global_space.c
  21. 2
      src/catalog/tde_keyring.c
  22. 12
      src/common/pg_tde_utils.c
  23. 5
      src/include/access/pg_tde_tdemap.h
  24. 3
      src/include/common/pg_tde_utils.h
  25. 12
      src/include/config.h
  26. 57
      src/include/config.h.in
  27. 2
      src/include/pg_tde_fe.h
  28. 3
      src/keyring/keyring_kmip_ereport.c

@ -31,10 +31,6 @@ jobs:
- name: Configure postgres - name: Configure postgres
run: ./configure run: ./configure
- name: Configure pg_tde
run: ./configure
working-directory: contrib/pg_tde
- name: Install perltidy - name: Install perltidy
run: sudo cpan -T SHANCOCK/Perl-Tidy-20230309.tar.gz run: sudo cpan -T SHANCOCK/Perl-Tidy-20230309.tar.gz

@ -76,7 +76,6 @@ jobs:
- name: Build pg_tde - name: Build pg_tde
run: | run: |
sudo -u postgres bash -c './configure'
sudo -u postgres bash -c 'make USE_PGXS=1' sudo -u postgres bash -c 'make USE_PGXS=1'
sudo make USE_PGXS=1 install sudo make USE_PGXS=1 install
working-directory: src/pg_tde working-directory: src/pg_tde

@ -65,7 +65,6 @@ jobs:
- name: Build pg_tde - name: Build pg_tde
run: | run: |
./configure
make -j MAJORVERSION=16 make -j MAJORVERSION=16
sudo make install sudo make install
working-directory: src/contrib/pg_tde working-directory: src/contrib/pg_tde

@ -65,7 +65,6 @@ jobs:
- name: Build pg_tde - name: Build pg_tde
run: | run: |
./configure
make -j MAJORVERSION=16 make -j MAJORVERSION=16
sudo make install sudo make install
working-directory: src/contrib/pg_tde working-directory: src/contrib/pg_tde

@ -65,7 +65,6 @@ jobs:
- name: Build pg_tde - name: Build pg_tde
run: | run: |
./configure
make -j MAJORVERSION=17 make -j MAJORVERSION=17
sudo make install sudo make install
working-directory: src/contrib/pg_tde working-directory: src/contrib/pg_tde

@ -71,7 +71,6 @@ jobs:
env: env:
POSTGRESQL_VERSION: ${{ matrix.postgresql-version }} POSTGRESQL_VERSION: ${{ matrix.postgresql-version }}
run: | run: |
sudo -u postgres bash -c './configure'
sudo -u postgres bash -c 'make USE_PGXS=1' sudo -u postgres bash -c 'make USE_PGXS=1'
sudo make USE_PGXS=1 MAJORVERSION=$POSTGRESQL_VERSION install sudo make USE_PGXS=1 MAJORVERSION=$POSTGRESQL_VERSION install
working-directory: src/pg_tde working-directory: src/pg_tde

2
.gitignore vendored

@ -5,11 +5,9 @@ __pycache__
/config.cache /config.cache
/config.log /config.log
/config.status /config.status
/Makefile
/autom4te.cache /autom4te.cache
/configure~ /configure~
t/results t/results
src/include/config.h
# tools files # tools files
typedefs-full.list typedefs-full.list

@ -64,8 +64,6 @@ src/libkmip/libkmip/src/kmip_bio.o \
src/libkmip/libkmip/src/kmip_locate.o \ src/libkmip/libkmip/src/kmip_locate.o \
src/libkmip/libkmip/src/kmip_memset.o src/libkmip/libkmip/src/kmip_memset.o
override PG_CPPFLAGS += @tde_CPPFLAGS@
ifdef USE_PGXS ifdef USE_PGXS
PG_CONFIG = pg_config PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs) PGXS := $(shell $(PG_CONFIG) --pgxs)
@ -79,7 +77,7 @@ include $(top_builddir)/src/Makefile.global
include $(top_srcdir)/contrib/contrib-global.mk include $(top_srcdir)/contrib/contrib-global.mk
endif endif
override SHLIB_LINK += @tde_LDFLAGS@ -lcrypto -lssl override SHLIB_LINK += -lcurl -lcrypto -lssl
# Fetches typedefs list for PostgreSQL core and merges it with typedefs defined in this project. # Fetches typedefs list for PostgreSQL core and merges it with typedefs defined in this project.
# https://wiki.postgresql.org/wiki/Running_pgindent_on_non-core_code_or_development_code # https://wiki.postgresql.org/wiki/Running_pgindent_on_non-core_code_or_development_code

@ -0,0 +1,20 @@
TDE_OBJS = \
src/access/pg_tde_tdemap.o \
src/access/pg_tde_xlog_encrypt.o \
src/catalog/tde_global_space.o \
src/catalog/tde_keyring.o \
src/catalog/tde_keyring_parse_opts.o \
src/catalog/tde_principal_key.o \
src/common/pg_tde_utils.o \
src/encryption/enc_aes.o \
src/encryption/enc_tde.o \
src/keyring/keyring_api.o \
src/keyring/keyring_curl.o \
src/keyring/keyring_file.o \
src/keyring/keyring_vault.o \
src/keyring/keyring_kmip.o \
src/keyring/keyring_kmip_ereport.o \
src/libkmip/libkmip/src/kmip.o \
src/libkmip/libkmip/src/kmip_bio.o \
src/libkmip/libkmip/src/kmip_locate.o \
src/libkmip/libkmip/src/kmip_memset.o

@ -71,7 +71,6 @@ Percona provides binary packages of `pg_tde` extension only for Percona Server f
```sh ```sh
cd pg_tde cd pg_tde
./configure
make USE_PGXS=1 make USE_PGXS=1
sudo make USE_PGXS=1 install sudo make USE_PGXS=1 install
``` ```

1807
config.guess vendored

File diff suppressed because it is too large Load Diff

1960
config.sub vendored

File diff suppressed because it is too large Load Diff

4984
configure vendored

File diff suppressed because it is too large Load Diff

@ -1,63 +0,0 @@
# Process this file with autoconf to produce a configure script.
# configure.ac
AC_INIT([pg_tde], [1.0.0-beta2], [https://github.com/percona/pg_tde/issues], [pg_tde])
AC_CANONICAL_HOST
# REQUIRE_LIB(name,lib,package,testfn,test_header_dir,test_include.h)
# name = The complete name of the library file without the extension.
# lib = The name of the library file without the 'lib' prefix and without the extension.
# package = The name of the package that holds the library
# testfn = One function included in the library that can be used for a test compilation.
# headerdir = The directory that contains required header.
# header = The header file to check for
# if there's a problem during the test compilation.
AC_DEFUN([REQUIRE_LIB], [ {
AC_ARG_WITH([$1], AS_HELP_STRING([--with-$1=<path>],[Location where $6 is installed]),[],[with_$1=default])
AS_IF([test "x$with_$1" == xdefault],
[
case $host_os in
darwin*) libpathx=($(brew --cellar)/$3/*)
tde_CPPFLAGS="$tde_CPPFLAGS -I$libpathx/include/$5"
tde_LDFLAGS="$tde_LDFLAGS -L$libpathx/lib -l$2" ;;
*) tde_CPPFLAGS="$tde_CPPFLAGS -I/usr/include/$5"
tde_LDFLAGS="$tde_LDFLAGS -l$2" ;;
esac
], [ #AS_ELSE
tde_CPPFLAGS="$tde_CPPFLAGS -I${with_$1}/include"
tde_LDFLAGS="$tde_LDFLAGS -L${with_$1}/lib -l$2"
])
LDFLAGS="$LDFLAGS $tde_LDFLAGS"
CPPFLAGS="$CPPFLAGS $tde_CPPFLAGS"
AC_CHECK_HEADER($6, [],
[
AC_MSG_ERROR([header file <$6> is required, try specifying --with-$1])
])
AC_CHECK_LIB($2, $4, [],
[
AC_MSG_ERROR([$1 was not found, try specifying --with-$1])
])
} ] )
AC_CONFIG_HEADERS(src/include/config.h)
#=======================================
REQUIRE_LIB(libcurl, curl, curl, curl_easy_setopt, curl, curl/curl.h)
case $host_os in
# on other systems libintl is a part of glibc
darwin*) REQUIRE_LIB(libintl, intl, gettext, ngettext, [], libintl.h)
esac
AC_SUBST(tde_CPPFLAGS)
AC_SUBST(tde_LDFLAGS)
AC_CONFIG_FILES([Makefile])
AC_OUTPUT

@ -13,8 +13,7 @@ WORKDIR /opt/pg_tde
COPY . . COPY . .
RUN ./configure && \ RUN make USE_PGXS=1 MAJORVERSION=16 && \
make USE_PGXS=1 MAJORVERSION=16 && \
make USE_PGXS=1 install make USE_PGXS=1 install
RUN cp /usr/share/postgresql/postgresql.conf.sample /etc/postgresql/postgresql.conf; \ RUN cp /usr/share/postgresql/postgresql.conf.sample /etc/postgresql/postgresql.conf; \
echo "shared_preload_libraries = 'pg_tde'" >> /etc/postgresql/postgresql.conf; \ echo "shared_preload_libraries = 'pg_tde'" >> /etc/postgresql/postgresql.conf; \

@ -55,7 +55,6 @@ To run the tests, use the following command:
``` ```
cd pg_tde cd pg_tde
./configure
make USE_PGXS=1 installcheck make USE_PGXS=1 installcheck
``` ```

@ -56,7 +56,6 @@ Install `pg_tde` using one of available installation methods:
``` ```
cd pg_tde cd pg_tde
./configure
make USE_PGXS=1 make USE_PGXS=1
sudo make USE_PGXS=1 install sudo make USE_PGXS=1 install
``` ```

@ -1,18 +1,6 @@
curldep = dependency('libcurl') curldep = dependency('libcurl')
pg_tde_version = '1.0.0-beta2'
conf_data = configuration_data()
conf_data.set_quoted('PACKAGE_NAME', 'pg_tde')
conf_data.set_quoted('PACKAGE_BUGREPORT', 'https://github.com/percona/pg_tde/issues')
conf_data.set_quoted('PACKAGE_VERSION', pg_tde_version)
conf_data.set_quoted('PACKAGE_STRING', 'pg_tde ' + pg_tde_version)
conf_data.set_quoted('PACKAGE_TARNAME', 'pg_tde')
configure_file(output : 'config.h',
configuration : conf_data)
pg_version = meson.project_version().substring(0,2) pg_version = meson.project_version().substring(0,2)
src_version = 'src' + pg_version src_version = 'src' + pg_version

@ -5,5 +5,3 @@
# We don't want to run pgindent on these files to avoid unnecessary conflicts. # We don't want to run pgindent on these files to avoid unnecessary conflicts.
src\d\d/ src\d\d/
# This file is generated by configure and should not be formatted
src/include/config.h

@ -63,7 +63,7 @@ TDEInitGlobalKeys(const char *dir)
RelKeyData *ikey; RelKeyData *ikey;
if (dir != NULL) if (dir != NULL)
pg_tde_set_globalspace_dir(dir); pg_tde_set_data_dir(dir);
ikey = pg_tde_get_key_from_file(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), TDE_KEY_TYPE_GLOBAL, false); ikey = pg_tde_get_key_from_file(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), TDE_KEY_TYPE_GLOBAL, false);
@ -94,11 +94,10 @@ init_default_keyring(void)
.provider_type = FILE_KEY_PROVIDER, .provider_type = FILE_KEY_PROVIDER,
}; };
if (getcwd(path, sizeof(path)) == NULL) char *data_path = make_absolute_path(PG_TDE_DATA_DIR);
elog(WARNING, "unable to get current working dir");
/* TODO: not sure about the location. Currently it's in $PGDATA */ join_path_components(path, data_path, KEYRING_DEFAULT_FILE_NAME);
join_path_components(path, PG_TDE_DATA_DIR, KEYRING_DEFAULT_FILE_NAME); free(data_path);
snprintf(provider.options, MAX_KEYRING_OPTION_LEN, snprintf(provider.options, MAX_KEYRING_OPTION_LEN,
"{" "{"

@ -689,7 +689,7 @@ debug_print_kerying(GenericKeyring *keyring)
static inline void static inline void
get_keyring_infofile_path(char *resPath, Oid dbOid) get_keyring_infofile_path(char *resPath, Oid dbOid)
{ {
join_path_components(resPath, PG_TDE_DATA_DIR, psprintf(PG_TDE_KEYRING_FILENAME, dbOid)); join_path_components(resPath, pg_tde_get_tde_data_dir(), psprintf(PG_TDE_KEYRING_FILENAME, dbOid));
} }
/* /*

@ -17,6 +17,7 @@
#include "miscadmin.h" #include "miscadmin.h"
#include "catalog/tde_principal_key.h" #include "catalog/tde_principal_key.h"
#include "access/pg_tde_tdemap.h" #include "access/pg_tde_tdemap.h"
#include "pg_tde.h"
#ifndef FRONTEND #ifndef FRONTEND
#include "access/genam.h" #include "access/genam.h"
@ -137,11 +138,18 @@ get_tde_tables_count(void)
#endif /* !FRONTEND */ #endif /* !FRONTEND */
static char globalspace_dir[MAXPGPATH] = {0}; static char globalspace_dir[MAXPGPATH] = PG_TDE_DATA_DIR;
void void
pg_tde_set_globalspace_dir(const char *dir) pg_tde_set_data_dir(const char *dir)
{ {
Assert(dir != NULL); Assert(dir != NULL);
strncpy(globalspace_dir, dir, sizeof(globalspace_dir)); strncpy(globalspace_dir, dir, sizeof(globalspace_dir));
} }
/* returns the palloc'd string */
char *
pg_tde_get_tde_data_dir()
{
return globalspace_dir;
}

@ -12,6 +12,7 @@
#include "utils/rel.h" #include "utils/rel.h"
#include "access/xlog_internal.h" #include "access/xlog_internal.h"
#include "catalog/tde_principal_key.h" #include "catalog/tde_principal_key.h"
#include "common/pg_tde_utils.h"
#include "storage/relfilelocator.h" #include "storage/relfilelocator.h"
/* Map entry flags */ /* Map entry flags */
@ -81,9 +82,9 @@ static inline void
pg_tde_set_db_file_paths(Oid dbOid, char *map_path, char *keydata_path) pg_tde_set_db_file_paths(Oid dbOid, char *map_path, char *keydata_path)
{ {
if (map_path) if (map_path)
join_path_components(map_path, PG_TDE_DATA_DIR, psprintf(PG_TDE_MAP_FILENAME, dbOid)); join_path_components(map_path, pg_tde_get_tde_data_dir(), psprintf(PG_TDE_MAP_FILENAME, dbOid));
if (keydata_path) if (keydata_path)
join_path_components(keydata_path, PG_TDE_DATA_DIR, psprintf(PG_TDE_KEYDATA_FILENAME, dbOid)); join_path_components(keydata_path, pg_tde_get_tde_data_dir(), psprintf(PG_TDE_KEYDATA_FILENAME, dbOid));
} }
const char *tde_sprint_key(InternalKey *k); const char *tde_sprint_key(InternalKey *k);

@ -19,5 +19,6 @@ extern List *get_all_tde_tables(void);
extern int get_tde_tables_count(void); extern int get_tde_tables_count(void);
#endif /* !FRONTEND */ #endif /* !FRONTEND */
extern void pg_tde_set_globalspace_dir(const char *dir); extern void pg_tde_set_data_dir(const char *dir);
extern char* pg_tde_get_tde_data_dir();
#endif /* PG_TDE_UTILS_H */ #endif /* PG_TDE_UTILS_H */

@ -0,0 +1,12 @@
#ifndef TDE_CONFIG_H
#define TDE_CONFIG_H
#define PACKAGE_NAME "pg_tde"
#define PACKAGE_VERSION "1.0.0-beta2"
#define PACKAGE_STRING PACKAGE_NAME" "PACKAGE_VERSION
#define PACKAGE_TARNAME "pg_tde"
#define PACKAGE_BUGREPORT "https://github.com/percona/pg_tde/issues"
#endif /* TDE_CONFIG_H */

@ -1,57 +0,0 @@
/* src/include/config.h.in. Generated from configure.ac by autoheader. */
/* Define to 1 if you have the <inttypes.h> header file. */
#undef HAVE_INTTYPES_H
/* Define to 1 if you have the `curl' library (-lcurl). */
#undef HAVE_LIBCURL
/* Define to 1 if you have the `intl' library (-lintl). */
#undef HAVE_LIBINTL
/* Define to 1 if you have the <stdint.h> header file. */
#undef HAVE_STDINT_H
/* Define to 1 if you have the <stdio.h> header file. */
#undef HAVE_STDIO_H
/* Define to 1 if you have the <stdlib.h> header file. */
#undef HAVE_STDLIB_H
/* Define to 1 if you have the <strings.h> header file. */
#undef HAVE_STRINGS_H
/* Define to 1 if you have the <string.h> header file. */
#undef HAVE_STRING_H
/* Define to 1 if you have the <sys/stat.h> header file. */
#undef HAVE_SYS_STAT_H
/* Define to 1 if you have the <sys/types.h> header file. */
#undef HAVE_SYS_TYPES_H
/* Define to 1 if you have the <unistd.h> header file. */
#undef HAVE_UNISTD_H
/* Define to the address where bug reports for this package should be sent. */
#undef PACKAGE_BUGREPORT
/* Define to the full name of this package. */
#undef PACKAGE_NAME
/* Define to the full name and version of this package. */
#undef PACKAGE_STRING
/* Define to the one symbol short name of this package. */
#undef PACKAGE_TARNAME
/* Define to the home page for this package. */
#undef PACKAGE_URL
/* Define to the version of this package. */
#undef PACKAGE_VERSION
/* Define to 1 if all of the C90 standard headers exist (not just the ones
required in a freestanding environment). This macro is provided for
backward compatibility; new code need not use it. */
#undef STDC_HEADERS

@ -76,7 +76,7 @@ static int tde_fe_error_level = 0;
#define LWLockAcquire(lock, mode) NULL #define LWLockAcquire(lock, mode) NULL
#define LWLockRelease(lock_files) NULL #define LWLockRelease(lock_files) NULL
#define LWLockHeldByMeInMode(lock, mode) NULL #define LWLockHeldByMeInMode(lock, mode) true
#define LWLock void #define LWLock void
#define LWLockMode void* #define LWLockMode void*
#define LW_SHARED NULL #define LW_SHARED NULL

@ -3,6 +3,9 @@
#include "keyring/keyring_kmip.h" #include "keyring/keyring_kmip.h"
#include "catalog/keyring_min.h" #include "catalog/keyring_min.h"
#ifdef FRONTEND
#include "pg_tde_fe.h"
#endif
void kmip_ereport(bool throw_error, const char *msg, int errCode) void kmip_ereport(bool throw_error, const char *msg, int errCode)
{ {

Loading…
Cancel
Save