open-dsi
/
postgres
mirror of https://github.com/postgres/postgres
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

PG-1710 Use temporary files instead of pipes for archive and restore commands

Browse Source 
For some to me unknown reason pgbackrest archive-push did not like
reading from a pipe so we create a temporary file instead. The location
for the temporary file is for now hardcoded to be /dev/shm since that is
guaranteed to be on a tmpfs mount on all common Linux distributions. We
might in the future want to make this configurable since it is a Linux
specific thing.

I tested this with PgBackRest 2.55.0.
pull/238/head
Andreas Karlsson 1 month ago committed by Andreas Karlsson
parent 9dc6c26cdf
commit fd6685348e
2 changed files with 75 additions and 91 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 85
      contrib/pg_tde/src/bin/pg_tde_archive_decrypt.c
  2. 81
      contrib/pg_tde/src/bin/pg_tde_restore_encrypt.c

85
contrib/pg_tde/src/bin/pg_tde_archive_decrypt.c
Unescape View File

@ -11,6 +11,8 @@
#include "access/pg_tde_fe_init.h"
#include "access/pg_tde_xlog_smgr.h"
#define TMPFS_DIRECTORY "/dev/shm"
static bool
is_segment(const char *filename)
{
@ -18,9 +20,10 @@ is_segment(const char *filename)
}
static void
write_decrypted_segment(const char *segpath, const char *segname, int pipewr)
write_decrypted_segment(const char *segpath, const char *segname, const char *tmppath)
{
int fd;
int segfd;
int tmpfd;
off_t fsize;
int r;
int w;
@ -29,10 +32,14 @@ write_decrypted_segment(const char *segpath, const char *segname, int pipewr)
PGAlignedXLogBlock buf;
off_t pos = 0;
fd = open(segpath, O_RDONLY | PG_BINARY, 0);
if (fd < 0)
segfd = open(segpath, O_RDONLY | PG_BINARY, 0);
if (segfd < 0)
pg_fatal("could not open file \"%s\": %m", segpath);
tmpfd = open(tmppath, O_CREAT | O_WRONLY | PG_BINARY, 0666);
if (tmpfd < 0)
pg_fatal("could not open file \"%s\": %m", tmppath);
/*
* WalSegSz extracted from the first page header but it might be
* encrypted. But we need to know the segment seize to decrypt it (it's
@ -40,10 +47,10 @@ write_decrypted_segment(const char *segpath, const char *segname, int pipewr)
* size from the file's actual size. XLogLongPageHeaderData->xlp_seg_size
* there is "just as a cross-check" anyway.
*/
fsize = lseek(fd, 0, SEEK_END);
fsize = lseek(segfd, 0, SEEK_END);
XLogFromFileName(segname, &tli, &segno, fsize);
r = xlog_smgr->seg_read(fd, buf.data, XLOG_BLCKSZ, pos, tli, segno, fsize);
r = xlog_smgr->seg_read(segfd, buf.data, XLOG_BLCKSZ, pos, tli, segno, fsize);
if (r == XLOG_BLCKSZ)
{
@ -73,16 +80,17 @@ write_decrypted_segment(const char *segpath, const char *segname, int pipewr)
pos += r;
w = write(pipewr, buf.data, XLOG_BLCKSZ);
w = write(tmpfd, buf.data, XLOG_BLCKSZ);
if (w < 0)
pg_fatal("could not write to pipe: %m");
pg_fatal("could not write file \"%s\": %m", tmppath);
else if (w != r)
pg_fatal("could not write to pipe: wrote %d of %d", w, r);
pg_fatal("could not write file \"%s\": wrote %d of %d",
tmppath, w, r);
while (1)
{
r = xlog_smgr->seg_read(fd, buf.data, XLOG_BLCKSZ, pos, tli, segno, fsize);
r = xlog_smgr->seg_read(segfd, buf.data, XLOG_BLCKSZ, pos, tli, segno, fsize);
if (r == 0)
break;
@ -91,15 +99,17 @@ write_decrypted_segment(const char *segpath, const char *segname, int pipewr)
pos += r;
w = write(pipewr, buf.data, r);
w = write(tmpfd, buf.data, r);
if (w < 0)
pg_fatal("could not write to pipe: %m");
pg_fatal("could not write file \"%s\": %m", tmppath);
else if (w != r)
pg_fatal("could not write to pipe: wrote %d of %d", w, r);
pg_fatal("could not write file \"%s\": wrote %d of %d",
tmppath, w, r);
}
close(fd);
close(tmpfd);
close(segfd);
}
static void
@ -119,10 +129,9 @@ main(int argc, char *argv[])
char *sourcepath;
char *sep;
char *sourcename;
char stdindir[MAXPGPATH] = "/tmp/pg_tde_archiveXXXXXX";
char stdinpath[MAXPGPATH];
char tmpdir[MAXPGPATH] = TMPFS_DIRECTORY "/pg_tde_archiveXXXXXX";
char tmppath[MAXPGPATH];
bool issegment;
int pipefd[2];
pid_t child;
int status;
int r;
@ -169,48 +178,29 @@ main(int argc, char *argv[])
{
char *s;
if (mkdtemp(stdindir) == NULL)
pg_fatal("could not create temporary directory \"%s\": %m", stdindir);
if (mkdtemp(tmpdir) == NULL)
pg_fatal("could not create temporary directory \"%s\": %m", tmpdir);
s = stpcpy(stdinpath, stdindir);
s = stpcpy(tmppath, tmpdir);
s = stpcpy(s, "/");
stpcpy(s, sourcename);
if (pipe(pipefd) < 0)
pg_fatal("could not create pipe: %m");
if (symlink("/dev/stdin", stdinpath) < 0)
pg_fatal("could not create symlink \"%s\": %m", stdinpath);
for (int i = 2; i < argc; i++)
if (strcmp(sourcepath, argv[i]) == 0)
argv[i] = stdinpath;
argv[i] = tmppath;
write_decrypted_segment(sourcepath, sourcename, tmppath);
}
child = fork();
if (child == 0)
{
if (issegment)
{
close(0);
dup2(pipefd[0], 0);
close(pipefd[0]);
close(pipefd[1]);
}
if (execvp(argv[2], argv + 2) < 0)
pg_fatal("exec failed: %m");
}
else if (child < 0)
pg_fatal("could not create background process: %m");
if (issegment)
{
close(pipefd[0]);
write_decrypted_segment(sourcepath, sourcename, pipefd[1]);
close(pipefd[1]);
}
r = waitpid(child, &status, 0);
if (r == (pid_t) -1)
pg_fatal("could not wait for child process: %m");
@ -219,10 +209,13 @@ main(int argc, char *argv[])
if (status != 0)
pg_fatal("%s", wait_result_to_str(status));
if (issegment && unlink(stdinpath) < 0)
pg_log_warning("could not remove symlink \"%s\": %m", stdinpath);
if (issegment && rmdir(stdindir) < 0)
pg_log_warning("could not remove directory \"%s\": %m", stdindir);
if (issegment)
{
if (unlink(tmppath) < 0)
pg_log_warning("could not remove file \"%s\": %m", tmppath);
if (rmdir(tmpdir) < 0)
pg_log_warning("could not remove directory \"%s\": %m", tmpdir);
}
return 0;
}

81
contrib/pg_tde/src/bin/pg_tde_restore_encrypt.c
Unescape View File

@ -11,6 +11,8 @@
#include "access/pg_tde_fe_init.h"
#include "access/pg_tde_xlog_smgr.h"
#define TMPFS_DIRECTORY "/dev/shm"
static bool
is_segment(const char *filename)
{
@ -18,9 +20,10 @@ is_segment(const char *filename)
}
static void
write_encrypted_segment(const char *segpath, const char *segname, int piperd)
write_encrypted_segment(const char *segpath, const char *segname, const char *tmppath)
{
int fd;
int tmpfd;
int segfd;
PGAlignedXLogBlock buf;
int r;
int w;
@ -30,17 +33,21 @@ write_encrypted_segment(const char *segpath, const char *segname, int piperd)
TimeLineID tli;
XLogSegNo segno;
fd = open(segpath, O_CREAT | O_WRONLY | PG_BINARY, 0666);
if (fd < 0)
tmpfd = open(tmppath, O_RDONLY | PG_BINARY, 0);
if (tmpfd < 0)
pg_fatal("could not open file \"%s\": %m", tmppath);
segfd = open(segpath, O_CREAT | O_WRONLY | PG_BINARY, 0666);
if (segfd < 0)
pg_fatal("could not open file \"%s\": %m", segpath);
r = read(piperd, buf.data, XLOG_BLCKSZ);
r = read(tmpfd, buf.data, XLOG_BLCKSZ);
if (r < 0)
pg_fatal("could not read from pipe: %m");
pg_fatal("could not read file \"%s\": %m", tmppath);
else if (r != XLOG_BLCKSZ)
pg_fatal("could not read from pipe: read %d of %d",
r, XLOG_BLCKSZ);
pg_fatal("could not read file \"%s\": read %d of %d",
tmppath, r, XLOG_BLCKSZ);
longhdr = (XLogLongPageHeader) buf.data;
walsegsz = longhdr->xlp_seg_size;
@ -59,7 +66,7 @@ write_encrypted_segment(const char *segpath, const char *segname, int piperd)
TDEXLogSmgrInitWriteReuseKey();
w = xlog_smgr->seg_write(fd, buf.data, r, pos, tli, segno, walsegsz);
w = xlog_smgr->seg_write(segfd, buf.data, r, pos, tli, segno, walsegsz);
if (w < 0)
pg_fatal("could not write file \"%s\": %m", segpath);
@ -71,14 +78,14 @@ write_encrypted_segment(const char *segpath, const char *segname, int piperd)
while (1)
{
r = read(piperd, buf.data, XLOG_BLCKSZ);
r = read(tmpfd, buf.data, XLOG_BLCKSZ);
if (r == 0)
break;
else if (r < 0)
pg_fatal("could not read from pipe: %m");
pg_fatal("could not read file \"%s\": %m", tmppath);
w = xlog_smgr->seg_write(fd, buf.data, r, pos, tli, segno, walsegsz);
w = xlog_smgr->seg_write(segfd, buf.data, r, pos, tli, segno, walsegsz);
if (w < 0)
pg_fatal("could not write file \"%s\": %m", segpath);
@ -89,7 +96,8 @@ write_encrypted_segment(const char *segpath, const char *segname, int piperd)
pos += w;
}
close(fd);
close(segfd);
close(tmpfd);
}
static void
@ -110,10 +118,9 @@ main(int argc, char *argv[])
char *targetpath;
char *sep;
char *targetname;
char stdoutdir[MAXPGPATH] = "/tmp/pg_tde_restoreXXXXXX";
char stdoutpath[MAXPGPATH];
char tmpdir[MAXPGPATH] = TMPFS_DIRECTORY "/pg_tde_restoreXXXXXX";
char tmppath[MAXPGPATH];
bool issegment;
int pipefd[2];
pid_t child;
int status;
int r;
@ -161,48 +168,27 @@ main(int argc, char *argv[])
{
char *s;
if (mkdtemp(stdoutdir) == NULL)
pg_fatal("could not create temporary directory \"%s\": %m", stdoutdir);
if (mkdtemp(tmpdir) == NULL)
pg_fatal("could not create temporary directory \"%s\": %m", tmpdir);
s = stpcpy(stdoutpath, stdoutdir);
s = stpcpy(tmppath, tmpdir);
s = stpcpy(s, "/");
stpcpy(s, targetname);
if (pipe(pipefd) < 0)
pg_fatal("could not create pipe: %m");
if (symlink("/dev/stdout", stdoutpath) < 0)
pg_fatal("could not create symlink \"%s\": %m", stdoutpath);
for (int i = 2; i < argc; i++)
if (strcmp(targetpath, argv[i]) == 0)
argv[i] = stdoutpath;
argv[i] = tmppath;
}
child = fork();
if (child == 0)
{
if (issegment)
{
close(1);
dup2(pipefd[1], 1);
close(pipefd[0]);
close(pipefd[1]);
}
if (execvp(argv[3], argv + 3) < 0)
pg_fatal("exec failed: %m");
}
else if (child < 0)
pg_fatal("could not create background process: %m");
if (issegment)
{
close(pipefd[1]);
write_encrypted_segment(targetpath, sourcename, pipefd[0]);
close(pipefd[0]);
}
r = waitpid(child, &status, 0);
if (r == (pid_t) -1)
pg_fatal("could not wait for child process: %m");
@ -211,10 +197,15 @@ main(int argc, char *argv[])
if (status != 0)
pg_fatal("%s", wait_result_to_str(status));
if (issegment && unlink(stdoutpath) < 0)
pg_log_warning("could not remove symlink \"%s\": %m", stdoutpath);
if (issegment && rmdir(stdoutdir) < 0)
pg_log_warning("could not remove directory \"%s\": %m", stdoutdir);
if (issegment)
{
write_encrypted_segment(targetpath, sourcename, tmppath);
if (unlink(tmppath) < 0)
pg_log_warning("could not remove file \"%s\": %m", tmppath);
if (rmdir(tmpdir) < 0)
pg_log_warning("could not remove directory \"%s\": %m", tmpdir);
}
return 0;
}

Write Preview
Loading…
Cancel
Save