CI updata 1: percona server for postgresql basic build and test

This action is similar to what we had before, executin tests on Ubuntu 22.04 with both meson and make, all tests passing. The test is refactored to easily support other configurations, but those are not part of this commit yet because of failures. Oters will be added in separate commits with required script/code changes. The commit also contains a compilation / possibly bug fix reported by newer GCC versions
8 months ago · 16cc2a1596
parent 9090f12201
commit 16cc2a1596
12 changed files with 127 additions and 132 deletions
--- a/.github/workflows/postgresql-current-make-debug.yml
+++ b/.github/workflows/postgresql-current-make-debug.yml
@ -1,60 +0,0 @@
-name: postgresql-current-make-debug
-on:
-  schedule:
-    - cron: "0 0 * * *"
-  pull_request:
-  workflow_dispatch:
-
-jobs:
-  build:
-    name: pg-current-make-debug
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
-            zlib1g-dev libssl-dev libpam0g-dev bison flex \
-            libipc-run-perl -y docbook-xsl docbook-xsl libxml2 libxml2-utils \
-            libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
-            libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
-            llvm-11 llvm-11-dev libselinux1-dev python3-dev \
-            uuid-dev liblz4-dev meson ninja-build  \
-            gpg wget libcurl4-openssl-dev libhttp-server-simple-perl
-          sudo /usr/bin/perl -MCPAN -e 'install IPC::Run'
-          sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
-          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
-          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
-          sudo apt update && sudo apt install -y vault
-
-      - name: Clone repository
-        uses: actions/checkout@v4
-        with:
-          path: 'src'
-          ref: ${{ github.ref }}
-
-      - name: Checkout submodules
-        run: |
-          git submodule update --init --recursive
-          cd contrib/pg_tde
-          git checkout main
-          git pull
-        working-directory: src
-
-      - name: Build postgres
-        run: |
-          ./configure --enable-debug --enable-cassert --enable-tap-tests 
-          make
-
-        working-directory: src
-
-      - name: Test postgres
-        run: |
-          TV=$(mktemp)
-          { exec >$TV; vault server -dev; } &
-          sleep 10
-          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
-          echo "Root token: $ROOT_TOKEN"
-
-          make check
-        working-directory: src
--- a/.github/workflows/postgresql-current-meson-debug.yml
+++ b/.github/workflows/postgresql-current-meson-debug.yml
@ -1,68 +0,0 @@
-name: postgresql-current-meson-debug
-on:
-  schedule:
-    - cron: "0 0 * * *"
-  pull_request:
-  workflow_dispatch:
-
-jobs:
-  build:
-    name: pg-current-meson-debug
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
-            zlib1g-dev libssl-dev libpam0g-dev bison flex \
-            libipc-run-perl -y docbook-xsl docbook-xsl libxml2 libxml2-utils \
-            libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
-            libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
-            llvm-11 llvm-11-dev libselinux1-dev python3-dev \
-            uuid-dev liblz4-dev meson ninja-build  \
-            gpg wget libcurl4-openssl-dev libhttp-server-simple-perl
-          sudo /usr/bin/perl -MCPAN -e 'install IPC::Run'
-          sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
-          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
-          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
-          sudo apt update && sudo apt install -y vault
-
-      - name: Clone repository
-        uses: actions/checkout@v4
-        with:
-          path: 'src'
-          ref: ${{ github.ref }}
-
-      - name: Checkout submodules
-        run: |
-          git submodule update --init --recursive
-          cd contrib/pg_tde
-          git checkout main
-          git pull
-        working-directory: src
-
-      - name: Build postgres
-        run: |
-          meson setup build --prefix `pwd`/../inst --buildtype=debug -Dcassert=true -Dtap_tests=enabled
-          cd build && ninja && ninja install
-        working-directory: src
-
-      - name: Test postgres
-        run: |
-          TV=$(mktemp)
-          { exec >$TV; vault server -dev; } &
-          sleep 10
-          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
-          echo "Root token: $ROOT_TOKEN"
-
-          meson test
-        working-directory: src/build
-
-      - name: Report on test fail
-        uses: actions/upload-artifact@v4
-        if: ${{ failure() }}
-        with:
-          name: Regressions diff and postgresql log
-          path: |
-            src/build/testrun/
-          retention-days: 3
--- a/.github/workflows/psp.yml
+++ b/.github/workflows/psp.yml
@ -0,0 +1,43 @@
+name: PSP
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Percona Postgres
+    strategy:
+      matrix:
+        ubuntu_version: [22]
+        build_type: [debug,debugoptimized]
+        build_script: [make, meson]
+    runs-on: ubuntu-${{ matrix.ubuntu_version }}.04
+    steps:
+
+      - name: Clone repository
+        uses: actions/checkout@v4
+        with:
+          path: 'src'
+          submodules: recursive
+          ref: ${{ github.ref }}
+
+      - name: Install dependencies
+        run: src/.scripts/ubuntu-deps.sh
+
+      - name: Setup kmip and vault
+        run: src/.scripts/setup-keyring-servers.sh
+
+      - name: Build and test postgres
+        run: src/.scripts/${{ matrix.build_script }}-build.sh ${{ matrix.build_type }}
+
+      - name: Test postgres
+        run: src/.scripts/${{ matrix.build_script }}-test.sh
+
+      - name: Report on test fail
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: testlog-ubuntu-${{ matrix.ubuntu_version }}.04-meson-${{ matrix.build_type }}
+          path: |
+            src/build/testrun/
+          retention-days: 3
--- a/.scripts/make-build.sh
+++ b/.scripts/make-build.sh
@ -0,0 +1,13 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+cd "$SCRIPT_DIR/../"
+
+if [ "$1" = "debugoptimized" ]; then
+    export CFLAGS="-O2"
+    export CXXFLAGS="-O2"
+fi
+
+./configure --enable-debug --enable-cassert --enable-tap-tests 
+make
--- a/.scripts/make-test.sh
+++ b/.scripts/make-test.sh
@ -0,0 +1,7 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+cd "$SCRIPT_DIR/../"
+
+make check-world
--- a/.scripts/meson-build.sh
+++ b/.scripts/meson-build.sh
@ -0,0 +1,8 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+cd "$SCRIPT_DIR/../"
+
+meson setup build --prefix `pwd`/../inst --buildtype=$1 -Dcassert=true -Dtap_tests=enabled
+cd build && ninja && ninja install
--- a/.scripts/meson-test.sh
+++ b/.scripts/meson-test.sh
@ -0,0 +1,7 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+cd "$SCRIPT_DIR/../build"
+
+meson test
--- a/.scripts/setup-keyring-servers.sh
+++ b/.scripts/setup-keyring-servers.sh
@ -0,0 +1,25 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+cd /tmp
+
+wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py
+python3 create_certificates.py
+cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem
+
+mkdir policies
+cd policies
+wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/examples/policy.json
+wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/examples/legacy_policy.json
+cd ..
+
+echo $SCRIPT_DIR
+pykmip-server -f "$SCRIPT_DIR/../contrib/pg_tde/pykmip-server.conf" -l /tmp/kmip-server.log &
+
+TV=$(mktemp)
+{ exec >$TV; vault server -dev; } &
+sleep 10
+ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+echo "ROOT_TOKEN=$ROOT_TOKEN"  >> $GITHUB_ENV
+echo "Root token: $ROOT_TOKEN"
--- a/.scripts/ubuntu-deps.sh
+++ b/.scripts/ubuntu-deps.sh
@ -0,0 +1,19 @@
+#!/bin/bash
+
+sudo apt update
+
+sudo apt install -y libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev bison flex libxml2 libxml2-utils libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev libsystemd-dev gettext tcl-dev libperl-dev pkg-config libselinux1-dev python3-dev uuid-dev liblz4-dev meson ninja-build gpg wget libcurl4-openssl-dev
+
+bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
+
+# Perl test dependencies
+
+sudo apt install -y libipc-run-perl python3-pykmip libhttp-server-simple-perl
+
+sudo /usr/bin/perl -MCPAN -e 'install IPC::Run'
+sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+
+# Vault
+wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+sudo apt update && sudo apt install -y vault
--- a/contrib/pg_tde/kmip-server.conf
+++ b/contrib/pg_tde/kmip-server.conf
@ -5,7 +5,7 @@ certificate_path=/tmp/server_certificate.pem
 key_path=/tmp/server_key.pem
 ca_path=/tmp/root_certificate.pem
 auth_suite=TLS1.2
-policy_path=/path/to/policy/file
+policy_path=/tmp/policies
 enable_tls_client_auth=True
 tls_cipher_suites=
    TLS_RSA_WITH_AES_128_CBC_SHA256
--- a/contrib/pg_tde/pykmip-server.conf
+++ b/contrib/pg_tde/pykmip-server.conf
@ -5,7 +5,7 @@ certificate_path=/tmp/server_certificate.pem
 key_path=/tmp/server_key.pem
 ca_path=/tmp/root_certificate.pem
 auth_suite=TLS1.2
-policy_path=/path/to/policy/file
+policy_path=/tmp/policies
 enable_tls_client_auth=True
 tls_cipher_suites=
    TLS_RSA_WITH_AES_128_CBC_SHA256
--- a/contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c
+++ b/contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c
@ -122,7 +122,7 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset)
 	char iv_prefix[16] = {0,};
 	size_t data_size = 0;
 	XLogPageHeader curr_page_hdr = &EncryptCurrentPageHrd;
-	XLogPageHeader enc_buf_page;
+	XLogPageHeader enc_buf_page = NULL;
 	RelKeyData *key = GetTdeGlobaleRelationKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID));
 	off_t enc_off;
 	size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ;
@ -175,7 +175,7 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset)
 		}

 		/*
-		 * The page is zeroed (no data), no sense to enctypt. This may happen
+		 * The page is zeroed (no data), no sense to encrypt. This may happen
 		 * when base_backup or other requests XLOG SWITCH and some pages in
 		 * XLog buffer still not used.
 		 */
@ -185,6 +185,7 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset)
 			Assert((*((char *) buf + enc_off) == 0) &&
 				   memcmp((char *) buf + enc_off, (char *) buf + enc_off + 1, data_size - 1) == 0);

+			enc_buf_page = (XLogPageHeader) (TDEXLogEncryptBuf + enc_off);
 			memcpy((char *) enc_buf_page, (char *) buf + enc_off, data_size);
 		}
 		else