From 16cc2a15961df01ca2e602f97c1bdf3a443db036 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Wed, 8 Jan 2025 20:53:57 +0000 Subject: [PATCH] CI updata 1: percona server for postgresql basic build and test This action is similar to what we had before, executin tests on Ubuntu 22.04 with both meson and make, all tests passing. The test is refactored to easily support other configurations, but those are not part of this commit yet because of failures. Oters will be added in separate commits with required script/code changes. The commit also contains a compilation / possibly bug fix reported by newer GCC versions --- .../postgresql-current-make-debug.yml | 60 ---------------- .../postgresql-current-meson-debug.yml | 68 ------------------- .github/workflows/psp.yml | 43 ++++++++++++ .scripts/make-build.sh | 13 ++++ .scripts/make-test.sh | 7 ++ .scripts/meson-build.sh | 8 +++ .scripts/meson-test.sh | 7 ++ .scripts/setup-keyring-servers.sh | 25 +++++++ .scripts/ubuntu-deps.sh | 19 ++++++ contrib/pg_tde/kmip-server.conf | 2 +- contrib/pg_tde/pykmip-server.conf | 2 +- .../pg_tde/src/access/pg_tde_xlog_encrypt.c | 5 +- 12 files changed, 127 insertions(+), 132 deletions(-) delete mode 100644 .github/workflows/postgresql-current-make-debug.yml delete mode 100644 .github/workflows/postgresql-current-meson-debug.yml create mode 100644 .github/workflows/psp.yml create mode 100755 .scripts/make-build.sh create mode 100755 .scripts/make-test.sh create mode 100755 .scripts/meson-build.sh create mode 100755 .scripts/meson-test.sh create mode 100755 .scripts/setup-keyring-servers.sh create mode 100755 .scripts/ubuntu-deps.sh diff --git a/.github/workflows/postgresql-current-make-debug.yml b/.github/workflows/postgresql-current-make-debug.yml deleted file mode 100644 index d056f222bad..00000000000 --- a/.github/workflows/postgresql-current-make-debug.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: postgresql-current-make-debug -on: - schedule: - - cron: "0 0 * * *" - pull_request: - workflow_dispatch: - -jobs: - build: - name: pg-current-make-debug - runs-on: ubuntu-22.04 - steps: - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \ - zlib1g-dev libssl-dev libpam0g-dev bison flex \ - libipc-run-perl -y docbook-xsl docbook-xsl libxml2 libxml2-utils \ - libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \ - libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \ - llvm-11 llvm-11-dev libselinux1-dev python3-dev \ - uuid-dev liblz4-dev meson ninja-build \ - gpg wget libcurl4-openssl-dev libhttp-server-simple-perl - sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' - sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' - wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list - sudo apt update && sudo apt install -y vault - - - name: Clone repository - uses: actions/checkout@v4 - with: - path: 'src' - ref: ${{ github.ref }} - - - name: Checkout submodules - run: | - git submodule update --init --recursive - cd contrib/pg_tde - git checkout main - git pull - working-directory: src - - - name: Build postgres - run: | - ./configure --enable-debug --enable-cassert --enable-tap-tests - make - - working-directory: src - - - name: Test postgres - run: | - TV=$(mktemp) - { exec >$TV; vault server -dev; } & - sleep 10 - export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) - echo "Root token: $ROOT_TOKEN" - - make check - working-directory: src diff --git a/.github/workflows/postgresql-current-meson-debug.yml b/.github/workflows/postgresql-current-meson-debug.yml deleted file mode 100644 index f3a98a58b24..00000000000 --- a/.github/workflows/postgresql-current-meson-debug.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: postgresql-current-meson-debug -on: - schedule: - - cron: "0 0 * * *" - pull_request: - workflow_dispatch: - -jobs: - build: - name: pg-current-meson-debug - runs-on: ubuntu-22.04 - steps: - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \ - zlib1g-dev libssl-dev libpam0g-dev bison flex \ - libipc-run-perl -y docbook-xsl docbook-xsl libxml2 libxml2-utils \ - libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \ - libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \ - llvm-11 llvm-11-dev libselinux1-dev python3-dev \ - uuid-dev liblz4-dev meson ninja-build \ - gpg wget libcurl4-openssl-dev libhttp-server-simple-perl - sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' - sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' - wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list - sudo apt update && sudo apt install -y vault - - - name: Clone repository - uses: actions/checkout@v4 - with: - path: 'src' - ref: ${{ github.ref }} - - - name: Checkout submodules - run: | - git submodule update --init --recursive - cd contrib/pg_tde - git checkout main - git pull - working-directory: src - - - name: Build postgres - run: | - meson setup build --prefix `pwd`/../inst --buildtype=debug -Dcassert=true -Dtap_tests=enabled - cd build && ninja && ninja install - working-directory: src - - - name: Test postgres - run: | - TV=$(mktemp) - { exec >$TV; vault server -dev; } & - sleep 10 - export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) - echo "Root token: $ROOT_TOKEN" - - meson test - working-directory: src/build - - - name: Report on test fail - uses: actions/upload-artifact@v4 - if: ${{ failure() }} - with: - name: Regressions diff and postgresql log - path: | - src/build/testrun/ - retention-days: 3 diff --git a/.github/workflows/psp.yml b/.github/workflows/psp.yml new file mode 100644 index 00000000000..723cabf4ab9 --- /dev/null +++ b/.github/workflows/psp.yml @@ -0,0 +1,43 @@ +name: PSP +on: + pull_request: + workflow_dispatch: + +jobs: + build: + name: Percona Postgres + strategy: + matrix: + ubuntu_version: [22] + build_type: [debug,debugoptimized] + build_script: [make, meson] + runs-on: ubuntu-${{ matrix.ubuntu_version }}.04 + steps: + + - name: Clone repository + uses: actions/checkout@v4 + with: + path: 'src' + submodules: recursive + ref: ${{ github.ref }} + + - name: Install dependencies + run: src/.scripts/ubuntu-deps.sh + + - name: Setup kmip and vault + run: src/.scripts/setup-keyring-servers.sh + + - name: Build and test postgres + run: src/.scripts/${{ matrix.build_script }}-build.sh ${{ matrix.build_type }} + + - name: Test postgres + run: src/.scripts/${{ matrix.build_script }}-test.sh + + - name: Report on test fail + uses: actions/upload-artifact@v4 + if: ${{ failure() }} + with: + name: testlog-ubuntu-${{ matrix.ubuntu_version }}.04-meson-${{ matrix.build_type }} + path: | + src/build/testrun/ + retention-days: 3 diff --git a/.scripts/make-build.sh b/.scripts/make-build.sh new file mode 100755 index 00000000000..ebf3d38c730 --- /dev/null +++ b/.scripts/make-build.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +cd "$SCRIPT_DIR/../" + +if [ "$1" = "debugoptimized" ]; then + export CFLAGS="-O2" + export CXXFLAGS="-O2" +fi + +./configure --enable-debug --enable-cassert --enable-tap-tests +make \ No newline at end of file diff --git a/.scripts/make-test.sh b/.scripts/make-test.sh new file mode 100755 index 00000000000..d7541a58b0e --- /dev/null +++ b/.scripts/make-test.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +cd "$SCRIPT_DIR/../" + +make check-world \ No newline at end of file diff --git a/.scripts/meson-build.sh b/.scripts/meson-build.sh new file mode 100755 index 00000000000..19885014ed6 --- /dev/null +++ b/.scripts/meson-build.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +cd "$SCRIPT_DIR/../" + +meson setup build --prefix `pwd`/../inst --buildtype=$1 -Dcassert=true -Dtap_tests=enabled +cd build && ninja && ninja install \ No newline at end of file diff --git a/.scripts/meson-test.sh b/.scripts/meson-test.sh new file mode 100755 index 00000000000..8d79bf5cd27 --- /dev/null +++ b/.scripts/meson-test.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +cd "$SCRIPT_DIR/../build" + +meson test \ No newline at end of file diff --git a/.scripts/setup-keyring-servers.sh b/.scripts/setup-keyring-servers.sh new file mode 100755 index 00000000000..b2a8a7327b6 --- /dev/null +++ b/.scripts/setup-keyring-servers.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +cd /tmp + +wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py +python3 create_certificates.py +cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem + +mkdir policies +cd policies +wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/examples/policy.json +wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/examples/legacy_policy.json +cd .. + +echo $SCRIPT_DIR +pykmip-server -f "$SCRIPT_DIR/../contrib/pg_tde/pykmip-server.conf" -l /tmp/kmip-server.log & + +TV=$(mktemp) +{ exec >$TV; vault server -dev; } & +sleep 10 +ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) +echo "ROOT_TOKEN=$ROOT_TOKEN" >> $GITHUB_ENV +echo "Root token: $ROOT_TOKEN" \ No newline at end of file diff --git a/.scripts/ubuntu-deps.sh b/.scripts/ubuntu-deps.sh new file mode 100755 index 00000000000..7d9b0f34d50 --- /dev/null +++ b/.scripts/ubuntu-deps.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +sudo apt update + +sudo apt install -y libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev bison flex libxml2 libxml2-utils libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev libsystemd-dev gettext tcl-dev libperl-dev pkg-config libselinux1-dev python3-dev uuid-dev liblz4-dev meson ninja-build gpg wget libcurl4-openssl-dev + +bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" + +# Perl test dependencies + +sudo apt install -y libipc-run-perl python3-pykmip libhttp-server-simple-perl + +sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' +sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' + +# Vault +wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list +sudo apt update && sudo apt install -y vault \ No newline at end of file diff --git a/contrib/pg_tde/kmip-server.conf b/contrib/pg_tde/kmip-server.conf index 7644e4b5952..ffea4f118ec 100644 --- a/contrib/pg_tde/kmip-server.conf +++ b/contrib/pg_tde/kmip-server.conf @@ -5,7 +5,7 @@ certificate_path=/tmp/server_certificate.pem key_path=/tmp/server_key.pem ca_path=/tmp/root_certificate.pem auth_suite=TLS1.2 -policy_path=/path/to/policy/file +policy_path=/tmp/policies enable_tls_client_auth=True tls_cipher_suites= TLS_RSA_WITH_AES_128_CBC_SHA256 diff --git a/contrib/pg_tde/pykmip-server.conf b/contrib/pg_tde/pykmip-server.conf index 7644e4b5952..ffea4f118ec 100644 --- a/contrib/pg_tde/pykmip-server.conf +++ b/contrib/pg_tde/pykmip-server.conf @@ -5,7 +5,7 @@ certificate_path=/tmp/server_certificate.pem key_path=/tmp/server_key.pem ca_path=/tmp/root_certificate.pem auth_suite=TLS1.2 -policy_path=/path/to/policy/file +policy_path=/tmp/policies enable_tls_client_auth=True tls_cipher_suites= TLS_RSA_WITH_AES_128_CBC_SHA256 diff --git a/contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c b/contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c index 5c40f73e902..a437ca9074a 100644 --- a/contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c +++ b/contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c @@ -122,7 +122,7 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset) char iv_prefix[16] = {0,}; size_t data_size = 0; XLogPageHeader curr_page_hdr = &EncryptCurrentPageHrd; - XLogPageHeader enc_buf_page; + XLogPageHeader enc_buf_page = NULL; RelKeyData *key = GetTdeGlobaleRelationKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID)); off_t enc_off; size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; @@ -175,7 +175,7 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset) } /* - * The page is zeroed (no data), no sense to enctypt. This may happen + * The page is zeroed (no data), no sense to encrypt. This may happen * when base_backup or other requests XLOG SWITCH and some pages in * XLog buffer still not used. */ @@ -185,6 +185,7 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset) Assert((*((char *) buf + enc_off) == 0) && memcmp((char *) buf + enc_off, (char *) buf + enc_off + 1, data_size - 1) == 0); + enc_buf_page = (XLogPageHeader) (TDEXLogEncryptBuf + enc_off); memcpy((char *) enc_buf_page, (char *) buf + enc_off, data_size); } else