mirror of https://github.com/postgres/postgres
Add option for parallel streaming of the transaction log while a base backup is running, to get the logfiles before the server has removed them. Also add a tool called pg_receivexlog, which streams the transaction log into files, creating a log archive without having to wait for segments to complete, thus decreasing the window of data loss without having to waste space using archive_timeout. This works best in combination with archive_command - suggested usage docs etc coming later.pull/1/head
Magnus Hagander
14 years ago
parent
2b64f3f17a
commit
d9bae53173
@ -0,0 +1,270 @@ |
||||
<!-- |
||||
doc/src/sgml/ref/pg_receivexlog.sgml |
||||
PostgreSQL documentation |
||||
--> |
||||
|
||||
<refentry id="app-pgreceivexlog"> |
||||
<refmeta> |
||||
<refentrytitle>pg_receivexlog</refentrytitle> |
||||
<manvolnum>1</manvolnum> |
||||
<refmiscinfo>Application</refmiscinfo> |
||||
</refmeta> |
||||
|
||||
<refnamediv> |
||||
<refname>pg_receivexlog</refname> |
||||
<refpurpose>streams transaction logs from a <productname>PostgreSQL</productname> cluster</refpurpose> |
||||
</refnamediv> |
||||
|
||||
<indexterm zone="app-pgreceivexlog"> |
||||
<primary>pg_receivexlog</primary> |
||||
</indexterm> |
||||
|
||||
<refsynopsisdiv> |
||||
<cmdsynopsis> |
||||
<command>pg_receivexlog</command> |
||||
<arg rep="repeat"><replaceable>option</></arg> |
||||
</cmdsynopsis> |
||||
</refsynopsisdiv> |
||||
|
||||
<refsect1> |
||||
<title> |
||||
Description |
||||
</title> |
||||
<para> |
||||
<application>pg_receivexlog</application> is used to stream transaction log |
||||
from a running <productname>PostgreSQL</productname> cluster. The transaction |
||||
log is streamed using the streaming replication protocol, and is written |
||||
to a local directory of files. This directory can be used as the archive |
||||
location for doing a restore using point-in-time recovery (see |
||||
<xref linkend="continuous-archiving">). |
||||
</para> |
||||
|
||||
<para> |
||||
<application>pg_receivexlog</application> streams the transaction |
||||
log in real time as it's being generated on the server, and does not wait |
||||
for segments to complete like <xref linkend="guc-archive-command"> does. |
||||
For this reason, it is not necessary to set |
||||
<xref linkend="guc-archive-timeout"> when using |
||||
<application>pg_receivexlog</application>. |
||||
</para> |
||||
|
||||
<para> |
||||
The transaction log is streamed over a regular |
||||
<productname>PostgreSQL</productname> connection, and uses the |
||||
replication protocol. The connection must be |
||||
made with a user having <literal>REPLICATION</literal> permissions (see |
||||
<xref linkend="role-attributes">), and the user must be granted explicit |
||||
permissions in <filename>pg_hba.conf</filename>. The server must also |
||||
be configured with <xref linkend="guc-max-wal-senders"> set high enough |
||||
to leave at least one session available for the stream. |
||||
</para> |
||||
</refsect1> |
||||
|
||||
<refsect1> |
||||
<title>Options</title> |
||||
|
||||
<para> |
||||
The following command-line options control the location and format of the |
||||
output. |
||||
|
||||
<variablelist> |
||||
<varlistentry> |
||||
<term><option>-D <replaceable class="parameter">directory</replaceable></option></term> |
||||
<term><option>--dir=<replaceable class="parameter">directory</replaceable></option></term> |
||||
<listitem> |
||||
<para> |
||||
Directory to write the output to. |
||||
</para> |
||||
<para> |
||||
This parameter is required. |
||||
</para> |
||||
</listitem> |
||||
</varlistentry> |
||||
</variablelist> |
||||
</para> |
||||
<para> |
||||
The following command-line options control the running of the program. |
||||
|
||||
<variablelist> |
||||
<varlistentry> |
||||
<term><option>-v</option></term> |
||||
<term><option>--verbose</option></term> |
||||
<listitem> |
||||
<para> |
||||
Enables verbose mode. |
||||
</para> |
||||
</listitem> |
||||
</varlistentry> |
||||
|
||||
</variablelist> |
||||
</para> |
||||
|
||||
<para> |
||||
The following command-line options control the database connection parameters. |
||||
|
||||
<variablelist> |
||||
<varlistentry> |
||||
<term><option>-s <replaceable class="parameter">interval</replaceable></option></term> |
||||
<term><option>--statusint=<replaceable class="parameter">interval</replaceable></option></term> |
||||
<listitem> |
||||
<para> |
||||
Specifies the number of seconds between status packets sent back to the |
||||
server. This is required if replication timeout is configured on the |
||||
server, and allows for easier monitoring. The default value is |
||||
10 seconds. |
||||
</para> |
||||
</listitem> |
||||
</varlistentry> |
||||
|
||||
<varlistentry> |
||||
<term><option>-h <replaceable class="parameter">host</replaceable></option></term> |
||||
<term><option>--host=<replaceable class="parameter">host</replaceable></option></term> |
||||
<listitem> |
||||
<para> |
||||
Specifies the host name of the machine on which the server is |
||||
running. If the value begins with a slash, it is used as the |
||||
directory for the Unix domain socket. The default is taken |
||||
from the <envar>PGHOST</envar> environment variable, if set, |
||||
else a Unix domain socket connection is attempted. |
||||
</para> |
||||
</listitem> |
||||
</varlistentry> |
||||
|
||||
<varlistentry> |
||||
<term><option>-p <replaceable class="parameter">port</replaceable></option></term> |
||||
<term><option>--port=<replaceable class="parameter">port</replaceable></option></term> |
||||
<listitem> |
||||
<para> |
||||
Specifies the TCP port or local Unix domain socket file |
||||
extension on which the server is listening for connections. |
||||
Defaults to the <envar>PGPORT</envar> environment variable, if |
||||
set, or a compiled-in default. |
||||
</para> |
||||
</listitem> |
||||
</varlistentry> |
||||
|
||||
<varlistentry> |
||||
<term><option>-U <replaceable>username</replaceable></option></term> |
||||
<term><option>--username=<replaceable class="parameter">username</replaceable></option></term> |
||||
<listitem> |
||||
<para> |
||||
User name to connect as. |
||||
</para> |
||||
</listitem> |
||||
</varlistentry> |
||||
|
||||
<varlistentry> |
||||
<term><option>-w</></term> |
||||
<term><option>--no-password</></term> |
||||
<listitem> |
||||
<para> |
||||
Never issue a password prompt. If the server requires |
||||
password authentication and a password is not available by |
||||
other means such as a <filename>.pgpass</filename> file, the |
||||
connection attempt will fail. This option can be useful in |
||||
batch jobs and scripts where no user is present to enter a |
||||
password. |
||||
</para> |
||||
</listitem> |
||||
</varlistentry> |
||||
|
||||
<varlistentry> |
||||
<term><option>-W</option></term> |
||||
<term><option>--password</option></term> |
||||
<listitem> |
||||
<para> |
||||
Force <application>pg_receivexlog</application> to prompt for a |
||||
password before connecting to a database. |
||||
</para> |
||||
|
||||
<para> |
||||
This option is never essential, since |
||||
<application>pg_receivexlog</application> will automatically prompt |
||||
for a password if the server demands password authentication. |
||||
However, <application>pg_receivexlog</application> will waste a |
||||
connection attempt finding out that the server wants a password. |
||||
In some cases it is worth typing <option>-W</> to avoid the extra |
||||
connection attempt. |
||||
</para> |
||||
</listitem> |
||||
</varlistentry> |
||||
</variablelist> |
||||
</para> |
||||
|
||||
<para> |
||||
Other, less commonly used, parameters are also available: |
||||
|
||||
<variablelist> |
||||
<varlistentry> |
||||
<term><option>-V</></term> |
||||
<term><option>--version</></term> |
||||
<listitem> |
||||
<para> |
||||
Print the <application>pg_receivexlog</application> version and exit. |
||||
</para> |
||||
</listitem> |
||||
</varlistentry> |
||||
|
||||
<varlistentry> |
||||
<term><option>-?</></term> |
||||
<term><option>--help</></term> |
||||
<listitem> |
||||
<para> |
||||
Show help about <application>pg_receivexlog</application> command line |
||||
arguments, and exit. |
||||
</para> |
||||
</listitem> |
||||
</varlistentry> |
||||
|
||||
</variablelist> |
||||
</para> |
||||
|
||||
</refsect1> |
||||
|
||||
<refsect1> |
||||
<title>Environment</title> |
||||
|
||||
<para> |
||||
This utility, like most other <productname>PostgreSQL</> utilities, |
||||
uses the environment variables supported by <application>libpq</> |
||||
(see <xref linkend="libpq-envars">). |
||||
</para> |
||||
|
||||
</refsect1> |
||||
|
||||
<refsect1> |
||||
<title>Notes</title> |
||||
|
||||
<para> |
||||
When using <application>pg_receivexlog</application> instead of |
||||
<xref linkend="guc-archive-command">, the server will continue to |
||||
recycle transaction log files even if the backups are not properly |
||||
archived, since there is no command that fails. This can be worked |
||||
around by having an <xref linkend="guc-archive-command"> that fails |
||||
when the file has not been properly archived yet. |
||||
</para> |
||||
|
||||
</refsect1> |
||||
|
||||
<refsect1> |
||||
<title>Examples</title> |
||||
|
||||
<para> |
||||
To stream the transaction log from the server at |
||||
<literal>mydbserver</literal> and store it in the local directory |
||||
<filename>/usr/local/pgsql/archive</filename>: |
||||
<screen> |
||||
<prompt>$</prompt> <userinput>pg_receivexlog -h mydbserver -D /home/pgbackup/archive</userinput> |
||||
</screen> |
||||
</para> |
||||
</refsect1> |
||||
|
||||
<refsect1> |
||||
<title>See Also</title> |
||||
|
||||
<simplelist type="inline"> |
||||
<member><xref linkend="APP-PGBASEBACKUP"></member> |
||||
</simplelist> |
||||
</refsect1> |
||||
|
||||
</refentry> |
||||
@ -1 +1,2 @@ |
||||
/pg_basebackup |
||||
/pg_receivexlog |
||||
@ -0,0 +1,465 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* pg_receivexlog.c - receive streaming transaction log data and write it |
||||
* to a local file. |
||||
* |
||||
* Author: Magnus Hagander <magnus@hagander.net> |
||||
* |
||||
* Portions Copyright (c) 1996-2011, PostgreSQL Global Development Group |
||||
* |
||||
* IDENTIFICATION |
||||
* src/bin/pg_basebackup/pg_receivexlog.c |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
|
||||
/*
|
||||
* We have to use postgres.h not postgres_fe.h here, because there's so much |
||||
* backend-only stuff in the XLOG include files we need. But we need a |
||||
* frontend-ish environment otherwise. Hence this ugly hack. |
||||
*/ |
||||
#define FRONTEND 1 |
||||
#include "postgres.h" |
||||
#include "libpq-fe.h" |
||||
#include "libpq/pqsignal.h" |
||||
#include "access/xlog_internal.h" |
||||
|
||||
#include "receivelog.h" |
||||
#include "streamutil.h" |
||||
|
||||
#include <dirent.h> |
||||
#include <sys/stat.h> |
||||
#include <sys/types.h> |
||||
#include <unistd.h> |
||||
|
||||
#include "getopt_long.h" |
||||
|
||||
/* Global options */ |
||||
char *basedir = NULL; |
||||
int verbose = 0; |
||||
int standby_message_timeout = 10; /* 10 sec = default */ |
||||
volatile bool time_to_abort = false; |
||||
|
||||
|
||||
static void usage(void); |
||||
static XLogRecPtr FindStreamingStart(XLogRecPtr currentpos, uint32 currenttimeline); |
||||
static void StreamLog(); |
||||
static bool segment_callback(XLogRecPtr segendpos, uint32 timeline); |
||||
|
||||
static void |
||||
usage(void) |
||||
{ |
||||
printf(_("%s receives PostgreSQL streaming transaction logs\n\n"), |
||||
progname); |
||||
printf(_("Usage:\n")); |
||||
printf(_(" %s [OPTION]...\n"), progname); |
||||
printf(_("\nOptions controlling the output:\n")); |
||||
printf(_(" -D, --dir=directory receive xlog files into this directory\n")); |
||||
printf(_("\nGeneral options:\n")); |
||||
printf(_(" -v, --verbose output verbose messages\n")); |
||||
printf(_(" -?, --help show this help, then exit\n")); |
||||
printf(_(" -V, --version output version information, then exit\n")); |
||||
printf(_("\nConnection options:\n")); |
||||
printf(_(" -s, --statusint=INTERVAL time between status packets sent to server (in seconds)\n")); |
||||
printf(_(" -h, --host=HOSTNAME database server host or socket directory\n")); |
||||
printf(_(" -p, --port=PORT database server port number\n")); |
||||
printf(_(" -U, --username=NAME connect as specified database user\n")); |
||||
printf(_(" -w, --no-password never prompt for password\n")); |
||||
printf(_(" -W, --password force password prompt (should happen automatically)\n")); |
||||
printf(_("\nReport bugs to <pgsql-bugs@postgresql.org>.\n")); |
||||
} |
||||
|
||||
static bool |
||||
segment_callback(XLogRecPtr segendpos, uint32 timeline) |
||||
{ |
||||
char fn[MAXPGPATH]; |
||||
struct stat statbuf; |
||||
|
||||
if (verbose) |
||||
fprintf(stderr, _("%s: finished segment at %X/%X (timeline %u)\n"), |
||||
progname, segendpos.xlogid, segendpos.xrecoff, timeline); |
||||
|
||||
/*
|
||||
* Check if there is a partial file for the name we just finished, and if |
||||
* there is, remove it under the assumption that we have now got all the |
||||
* data we need. |
||||
*/ |
||||
segendpos.xrecoff /= XLOG_SEG_SIZE; |
||||
PrevLogSeg(segendpos.xlogid, segendpos.xrecoff); |
||||
snprintf(fn, sizeof(fn), "%s/%08X%08X%08X.partial", |
||||
basedir, timeline, |
||||
segendpos.xlogid, |
||||
segendpos.xrecoff); |
||||
if (stat(fn, &statbuf) == 0) |
||||
{ |
||||
/* File existed, get rid of it */ |
||||
if (verbose) |
||||
fprintf(stderr, _("%s: removing file \"%s\"\n"), |
||||
progname, fn); |
||||
unlink(fn); |
||||
} |
||||
|
||||
/*
|
||||
* Never abort from this - we handle all aborting in continue_streaming() |
||||
*/ |
||||
return false; |
||||
} |
||||
|
||||
static bool |
||||
continue_streaming(void) |
||||
{ |
||||
if (time_to_abort) |
||||
{ |
||||
fprintf(stderr, _("%s: received interrupt signal, exiting.\n"), |
||||
progname); |
||||
return true; |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
/*
|
||||
* Determine starting location for streaming, based on: |
||||
* 1. If there are existing xlog segments, start at the end of the last one |
||||
* 2. If the last one is a partial segment, rename it and start over, since |
||||
* we don't sync after every write. |
||||
* 3. If no existing xlog exists, start from the beginning of the current |
||||
* WAL segment. |
||||
*/ |
||||
static XLogRecPtr |
||||
FindStreamingStart(XLogRecPtr currentpos, uint32 currenttimeline) |
||||
{ |
||||
DIR *dir; |
||||
struct dirent *dirent; |
||||
int i; |
||||
bool b; |
||||
uint32 high_log = 0; |
||||
uint32 high_seg = 0; |
||||
bool partial = false; |
||||
|
||||
dir = opendir(basedir); |
||||
if (dir == NULL) |
||||
{ |
||||
fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"), |
||||
progname, basedir, strerror(errno)); |
||||
disconnect_and_exit(1); |
||||
} |
||||
|
||||
while ((dirent = readdir(dir)) != NULL) |
||||
{ |
||||
char fullpath[MAXPGPATH]; |
||||
struct stat statbuf; |
||||
uint32 tli, |
||||
log, |
||||
seg; |
||||
|
||||
if (!strcmp(dirent->d_name, ".") || !strcmp(dirent->d_name, "..")) |
||||
continue; |
||||
|
||||
/* xlog files are always 24 characters */ |
||||
if (strlen(dirent->d_name) != 24) |
||||
continue; |
||||
|
||||
/* Filenames are always made out of 0-9 and A-F */ |
||||
b = false; |
||||
for (i = 0; i < 24; i++) |
||||
{ |
||||
if (!(dirent->d_name[i] >= '0' && dirent->d_name[i] <= '9') && |
||||
!(dirent->d_name[i] >= 'A' && dirent->d_name[i] <= 'F')) |
||||
{ |
||||
b = true; |
||||
break; |
||||
} |
||||
} |
||||
if (b) |
||||
continue; |
||||
|
||||
/*
|
||||
* Looks like an xlog file. Parse its position. |
||||
*/ |
||||
if (sscanf(dirent->d_name, "%08X%08X%08X", &tli, &log, &seg) != 3) |
||||
{ |
||||
fprintf(stderr, _("%s: could not parse xlog filename \"%s\"\n"), |
||||
progname, dirent->d_name); |
||||
disconnect_and_exit(1); |
||||
} |
||||
|
||||
/* Ignore any files that are for another timeline */ |
||||
if (tli != currenttimeline) |
||||
continue; |
||||
|
||||
/* Check if this is a completed segment or not */ |
||||
snprintf(fullpath, sizeof(fullpath), "%s/%s", basedir, dirent->d_name); |
||||
if (stat(fullpath, &statbuf) != 0) |
||||
{ |
||||
fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), |
||||
progname, fullpath, strerror(errno)); |
||||
disconnect_and_exit(1); |
||||
} |
||||
|
||||
if (statbuf.st_size == 16 * 1024 * 1024) |
||||
{ |
||||
/* Completed segment */ |
||||
if (log > high_log || |
||||
(log == high_log && seg > high_seg)) |
||||
{ |
||||
high_log = log; |
||||
high_seg = seg; |
||||
continue; |
||||
} |
||||
} |
||||
else |
||||
{ |
||||
/*
|
||||
* This is a partial file. Rename it out of the way. |
||||
*/ |
||||
char newfn[MAXPGPATH]; |
||||
|
||||
fprintf(stderr, _("%s: renaming partial file \"%s\" to \"%s.partial\"\n"), |
||||
progname, dirent->d_name, dirent->d_name); |
||||
|
||||
snprintf(newfn, sizeof(newfn), "%s/%s.partial", |
||||
basedir, dirent->d_name); |
||||
|
||||
if (stat(newfn, &statbuf) == 0) |
||||
{ |
||||
/*
|
||||
* XXX: perhaps we should only error out if the existing file |
||||
* is larger? |
||||
*/ |
||||
fprintf(stderr, _("%s: file \"%s\" already exists. Check and clean up manually.\n"), |
||||
progname, newfn); |
||||
disconnect_and_exit(1); |
||||
} |
||||
if (rename(fullpath, newfn) != 0) |
||||
{ |
||||
fprintf(stderr, _("%s: could not rename \"%s\" to \"%s\": %s\n"), |
||||
progname, fullpath, newfn, strerror(errno)); |
||||
disconnect_and_exit(1); |
||||
} |
||||
|
||||
/* Don't continue looking for more, we assume this is the last */ |
||||
partial = true; |
||||
break; |
||||
} |
||||
} |
||||
|
||||
closedir(dir); |
||||
|
||||
if (high_log > 0 || high_seg > 0) |
||||
{ |
||||
XLogRecPtr high_ptr; |
||||
|
||||
if (!partial) |
||||
{ |
||||
/*
|
||||
* If the segment was partial, the pointer is already at the right |
||||
* location since we want to re-transmit that segment. If it was |
||||
* not, we need to move it to the next segment, since we are |
||||
* tracking the last one that was complete. |
||||
*/ |
||||
NextLogSeg(high_log, high_seg); |
||||
} |
||||
|
||||
high_ptr.xlogid = high_log; |
||||
high_ptr.xrecoff = high_seg * XLOG_SEG_SIZE; |
||||
|
||||
return high_ptr; |
||||
} |
||||
else |
||||
return currentpos; |
||||
} |
||||
|
||||
/*
|
||||
* Start the log streaming |
||||
*/ |
||||
static void |
||||
StreamLog(void) |
||||
{ |
||||
PGresult *res; |
||||
uint32 timeline; |
||||
XLogRecPtr startpos; |
||||
|
||||
/*
|
||||
* Connect in replication mode to the server |
||||
*/ |
||||
conn = GetConnection(); |
||||
|
||||
/*
|
||||
* Run IDENTIFY_SYSTEM so we can get the timeline and current xlog |
||||
* position. |
||||
*/ |
||||
res = PQexec(conn, "IDENTIFY_SYSTEM"); |
||||
if (PQresultStatus(res) != PGRES_TUPLES_OK) |
||||
{ |
||||
fprintf(stderr, _("%s: could not identify system: %s\n"), |
||||
progname, PQerrorMessage(conn)); |
||||
disconnect_and_exit(1); |
||||
} |
||||
if (PQntuples(res) != 1) |
||||
{ |
||||
fprintf(stderr, _("%s: could not identify system, got %i rows\n"), |
||||
progname, PQntuples(res)); |
||||
disconnect_and_exit(1); |
||||
} |
||||
timeline = atoi(PQgetvalue(res, 0, 1)); |
||||
if (sscanf(PQgetvalue(res, 0, 2), "%X/%X", &startpos.xlogid, &startpos.xrecoff) != 2) |
||||
{ |
||||
fprintf(stderr, _("%s: could not parse log start position from value \"%s\"\n"), |
||||
progname, PQgetvalue(res, 0, 2)); |
||||
disconnect_and_exit(1); |
||||
} |
||||
PQclear(res); |
||||
|
||||
/*
|
||||
* Figure out where to start streaming. |
||||
*/ |
||||
startpos = FindStreamingStart(startpos, timeline); |
||||
|
||||
/*
|
||||
* Always start streaming at the beginning of a segment |
||||
*/ |
||||
startpos.xrecoff -= startpos.xrecoff % XLOG_SEG_SIZE; |
||||
|
||||
/*
|
||||
* Start the replication |
||||
*/ |
||||
if (verbose) |
||||
fprintf(stderr, _("%s: starting log streaming at %X/%X (timeline %u)\n"), |
||||
progname, startpos.xlogid, startpos.xrecoff, timeline); |
||||
|
||||
ReceiveXlogStream(conn, startpos, timeline, NULL, basedir, |
||||
segment_callback, continue_streaming, |
||||
standby_message_timeout); |
||||
} |
||||
|
||||
/*
|
||||
* When sigint is called, just tell the system to exit at the next possible |
||||
* moment. |
||||
*/ |
||||
static void |
||||
sigint_handler(int signum) |
||||
{ |
||||
time_to_abort = true; |
||||
} |
||||
|
||||
int |
||||
main(int argc, char **argv) |
||||
{ |
||||
static struct option long_options[] = { |
||||
{"help", no_argument, NULL, '?'}, |
||||
{"version", no_argument, NULL, 'V'}, |
||||
{"dir", required_argument, NULL, 'D'}, |
||||
{"host", required_argument, NULL, 'h'}, |
||||
{"port", required_argument, NULL, 'p'}, |
||||
{"username", required_argument, NULL, 'U'}, |
||||
{"no-password", no_argument, NULL, 'w'}, |
||||
{"password", no_argument, NULL, 'W'}, |
||||
{"statusint", required_argument, NULL, 's'}, |
||||
{"verbose", no_argument, NULL, 'v'}, |
||||
{NULL, 0, NULL, 0} |
||||
}; |
||||
int c; |
||||
|
||||
int option_index; |
||||
|
||||
progname = get_progname(argv[0]); |
||||
set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_receivexlog")); |
||||
|
||||
if (argc > 1) |
||||
{ |
||||
if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) |
||||
{ |
||||
usage(); |
||||
exit(0); |
||||
} |
||||
else if (strcmp(argv[1], "-V") == 0 |
||||
|| strcmp(argv[1], "--version") == 0) |
||||
{ |
||||
puts("pg_receivexlog (PostgreSQL) " PG_VERSION); |
||||
exit(0); |
||||
} |
||||
} |
||||
|
||||
while ((c = getopt_long(argc, argv, "D:h:p:U:s:wWv", |
||||
long_options, &option_index)) != -1) |
||||
{ |
||||
switch (c) |
||||
{ |
||||
case 'D': |
||||
basedir = xstrdup(optarg); |
||||
break; |
||||
case 'h': |
||||
dbhost = xstrdup(optarg); |
||||
break; |
||||
case 'p': |
||||
if (atoi(optarg) <= 0) |
||||
{ |
||||
fprintf(stderr, _("%s: invalid port number \"%s\"\n"), |
||||
progname, optarg); |
||||
exit(1); |
||||
} |
||||
dbport = xstrdup(optarg); |
||||
break; |
||||
case 'U': |
||||
dbuser = xstrdup(optarg); |
||||
break; |
||||
case 'w': |
||||
dbgetpassword = -1; |
||||
break; |
||||
case 'W': |
||||
dbgetpassword = 1; |
||||
break; |
||||
case 's': |
||||
standby_message_timeout = atoi(optarg); |
||||
if (standby_message_timeout < 0) |
||||
{ |
||||
fprintf(stderr, _("%s: invalid status interval \"%s\"\n"), |
||||
progname, optarg); |
||||
exit(1); |
||||
} |
||||
break; |
||||
case 'v': |
||||
verbose++; |
||||
break; |
||||
default: |
||||
|
||||
/*
|
||||
* getopt_long already emitted a complaint |
||||
*/ |
||||
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), |
||||
progname); |
||||
exit(1); |
||||
} |
||||
} |
||||
|
||||
/*
|
||||
* Any non-option arguments? |
||||
*/ |
||||
if (optind < argc) |
||||
{ |
||||
fprintf(stderr, |
||||
_("%s: too many command-line arguments (first is \"%s\")\n"), |
||||
progname, argv[optind]); |
||||
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), |
||||
progname); |
||||
exit(1); |
||||
} |
||||
|
||||
/*
|
||||
* Required arguments |
||||
*/ |
||||
if (basedir == NULL) |
||||
{ |
||||
fprintf(stderr, _("%s: no target directory specified\n"), progname); |
||||
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), |
||||
progname); |
||||
exit(1); |
||||
} |
||||
|
||||
#ifndef WIN32 |
||||
pqsignal(SIGINT, sigint_handler); |
||||
#endif |
||||
|
||||
StreamLog(); |
||||
|
||||
exit(0); |
||||
} |
||||
@ -0,0 +1,398 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* receivelog.c - receive transaction log files using the streaming |
||||
* replication protocol. |
||||
* |
||||
* Author: Magnus Hagander <magnus@hagander.net> |
||||
* |
||||
* Portions Copyright (c) 1996-2011, PostgreSQL Global Development Group |
||||
* |
||||
* IDENTIFICATION |
||||
* src/bin/pg_basebackup/receivelog.c |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
|
||||
/*
|
||||
* We have to use postgres.h not postgres_fe.h here, because there's so much |
||||
* backend-only stuff in the XLOG include files we need. But we need a |
||||
* frontend-ish environment otherwise. Hence this ugly hack. |
||||
*/ |
||||
#define FRONTEND 1 |
||||
#include "postgres.h" |
||||
#include "libpq-fe.h" |
||||
#include "access/xlog_internal.h" |
||||
#include "replication/walprotocol.h" |
||||
#include "utils/datetime.h" |
||||
|
||||
#include "receivelog.h" |
||||
#include "streamutil.h" |
||||
|
||||
#include <sys/time.h> |
||||
#include <sys/types.h> |
||||
#include <unistd.h> |
||||
|
||||
|
||||
/* Size of the streaming replication protocol header */ |
||||
#define STREAMING_HEADER_SIZE (1+8+8+8) |
||||
|
||||
const XLogRecPtr InvalidXLogRecPtr = {0, 0}; |
||||
|
||||
/*
|
||||
* Open a new WAL file in the specified directory. Store the name |
||||
* (not including the full directory) in namebuf. Assumes there is |
||||
* enough room in this buffer... |
||||
*/ |
||||
static int |
||||
open_walfile(XLogRecPtr startpoint, uint32 timeline, char *basedir, char *namebuf) |
||||
{ |
||||
int f; |
||||
char fn[MAXPGPATH]; |
||||
|
||||
XLogFileName(namebuf, timeline, startpoint.xlogid, |
||||
startpoint.xrecoff / XLOG_SEG_SIZE); |
||||
|
||||
snprintf(fn, sizeof(fn), "%s/%s", basedir, namebuf); |
||||
f = open(fn, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY, 0666); |
||||
if (f == -1) |
||||
fprintf(stderr, _("%s: Could not open WAL segment %s: %s\n"), |
||||
progname, namebuf, strerror(errno)); |
||||
return f; |
||||
} |
||||
|
||||
/*
|
||||
* Local version of GetCurrentTimestamp(), since we are not linked with |
||||
* backend code. |
||||
*/ |
||||
static TimestampTz |
||||
localGetCurrentTimestamp(void) |
||||
{ |
||||
TimestampTz result; |
||||
struct timeval tp; |
||||
|
||||
gettimeofday(&tp, NULL); |
||||
|
||||
result = (TimestampTz) tp.tv_sec - |
||||
((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY); |
||||
|
||||
#ifdef HAVE_INT64_TIMESTAMP |
||||
result = (result * USECS_PER_SEC) + tp.tv_usec; |
||||
#else |
||||
result = result + (tp.tv_usec / 1000000.0); |
||||
#endif |
||||
|
||||
return result; |
||||
} |
||||
|
||||
/*
|
||||
* Receive a log stream starting at the specified position. |
||||
* |
||||
* If sysidentifier is specified, validate that both the system |
||||
* identifier and the timeline matches the specified ones |
||||
* (by sending an extra IDENTIFY_SYSTEM command) |
||||
* |
||||
* All received segments will be written to the directory |
||||
* specified by basedir. |
||||
* |
||||
* The segment_finish callback will be called after each segment |
||||
* has been finished, and the stream_continue callback will be |
||||
* called every time data is received. If either of these callbacks |
||||
* return true, the streaming will stop and the function |
||||
* return. As long as they return false, streaming will continue |
||||
* indefinitely. |
||||
* |
||||
* standby_message_timeout controls how often we send a message |
||||
* back to the master letting it know our progress, in seconds. |
||||
* This message will only contain the write location, and never |
||||
* flush or replay. |
||||
* |
||||
* Note: The log position *must* be at a log segment start! |
||||
*/ |
||||
bool |
||||
ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline, char *sysidentifier, char *basedir, segment_finish_callback segment_finish, stream_continue_callback stream_continue, int standby_message_timeout) |
||||
{ |
||||
char query[128]; |
||||
char current_walfile_name[MAXPGPATH]; |
||||
PGresult *res; |
||||
char *copybuf = NULL; |
||||
int walfile = -1; |
||||
int64 last_status = -1; |
||||
XLogRecPtr blockpos = InvalidXLogRecPtr; |
||||
|
||||
if (sysidentifier != NULL) |
||||
{ |
||||
/* Validate system identifier and timeline hasn't changed */ |
||||
res = PQexec(conn, "IDENTIFY_SYSTEM"); |
||||
if (PQresultStatus(res) != PGRES_TUPLES_OK) |
||||
{ |
||||
fprintf(stderr, _("%s: could not identify system: %s\n"), |
||||
progname, PQerrorMessage(conn)); |
||||
PQclear(res); |
||||
return false; |
||||
} |
||||
if (strcmp(sysidentifier, PQgetvalue(res, 0, 0)) != 0) |
||||
{ |
||||
fprintf(stderr, _("%s: system identifier does not match between base backup and streaming connection\n"), progname); |
||||
PQclear(res); |
||||
return false; |
||||
} |
||||
if (timeline != atoi(PQgetvalue(res, 0, 1))) |
||||
{ |
||||
fprintf(stderr, _("%s: timeline does not match between base backup and streaming connection\n"), progname); |
||||
PQclear(res); |
||||
return false; |
||||
} |
||||
PQclear(res); |
||||
} |
||||
|
||||
/* Initiate the replication stream at specified location */ |
||||
snprintf(query, sizeof(query), "START_REPLICATION %X/%X", startpos.xlogid, startpos.xrecoff); |
||||
res = PQexec(conn, query); |
||||
if (PQresultStatus(res) != PGRES_COPY_BOTH) |
||||
{ |
||||
fprintf(stderr, _("%s: could not start replication: %s\n"), |
||||
progname, PQresultErrorMessage(res)); |
||||
return false; |
||||
} |
||||
PQclear(res); |
||||
|
||||
/*
|
||||
* Receive the actual xlog data |
||||
*/ |
||||
while (1) |
||||
{ |
||||
int r; |
||||
int xlogoff; |
||||
int bytes_left; |
||||
int bytes_written; |
||||
int64 now; |
||||
|
||||
if (copybuf != NULL) |
||||
{ |
||||
PQfreemem(copybuf); |
||||
copybuf = NULL; |
||||
} |
||||
|
||||
/*
|
||||
* Check if we should continue streaming, or abort at this point. |
||||
*/ |
||||
if (stream_continue && stream_continue()) |
||||
{ |
||||
if (walfile != -1) |
||||
{ |
||||
fsync(walfile); |
||||
close(walfile); |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
/*
|
||||
* Potentially send a status message to the master |
||||
*/ |
||||
now = localGetCurrentTimestamp(); |
||||
if (standby_message_timeout > 0 && |
||||
last_status < now - standby_message_timeout * 1000000) |
||||
{ |
||||
/* Time to send feedback! */ |
||||
char replybuf[sizeof(StandbyReplyMessage) + 1]; |
||||
StandbyReplyMessage *replymsg = (StandbyReplyMessage *) (replybuf + 1); |
||||
|
||||
replymsg->write = blockpos; |
||||
replymsg->flush = InvalidXLogRecPtr; |
||||
replymsg->apply = InvalidXLogRecPtr; |
||||
replymsg->sendTime = now; |
||||
replybuf[0] = 'r'; |
||||
|
||||
if (PQputCopyData(conn, replybuf, sizeof(replybuf)) <= 0 || |
||||
PQflush(conn)) |
||||
{ |
||||
fprintf(stderr, _("%s: could not send feedback packet: %s"), |
||||
progname, PQerrorMessage(conn)); |
||||
return false; |
||||
} |
||||
|
||||
last_status = now; |
||||
} |
||||
|
||||
r = PQgetCopyData(conn, ©buf, 1); |
||||
if (r == 0) |
||||
{ |
||||
/*
|
||||
* In async mode, and no data available. We block on reading but |
||||
* not more than the specified timeout, so that we can send a |
||||
* response back to the client. |
||||
*/ |
||||
fd_set input_mask; |
||||
struct timeval timeout; |
||||
struct timeval *timeoutptr; |
||||
|
||||
FD_ZERO(&input_mask); |
||||
FD_SET(PQsocket(conn), &input_mask); |
||||
if (standby_message_timeout) |
||||
{ |
||||
timeout.tv_sec = last_status + standby_message_timeout - now - 1; |
||||
if (timeout.tv_sec <= 0) |
||||
timeout.tv_sec = 1; /* Always sleep at least 1 sec */ |
||||
timeout.tv_usec = 0; |
||||
timeoutptr = &timeout; |
||||
} |
||||
else |
||||
timeoutptr = NULL; |
||||
|
||||
r = select(PQsocket(conn) + 1, &input_mask, NULL, NULL, timeoutptr); |
||||
if (r == 0 || (r < 0 && errno == EINTR)) |
||||
{ |
||||
/*
|
||||
* Got a timeout or signal. Continue the loop and either |
||||
* deliver a status packet to the server or just go back into |
||||
* blocking. |
||||
*/ |
||||
continue; |
||||
} |
||||
else if (r < 0) |
||||
{ |
||||
fprintf(stderr, _("%s: select() failed: %m\n"), progname); |
||||
return false; |
||||
} |
||||
/* Else there is actually data on the socket */ |
||||
if (PQconsumeInput(conn) == 0) |
||||
{ |
||||
fprintf(stderr, _("%s: could not receive data from WAL stream: %s\n"), |
||||
progname, PQerrorMessage(conn)); |
||||
return false; |
||||
} |
||||
continue; |
||||
} |
||||
if (r == -1) |
||||
/* End of copy stream */ |
||||
break; |
||||
if (r == -2) |
||||
{ |
||||
fprintf(stderr, _("%s: could not read copy data: %s\n"), |
||||
progname, PQerrorMessage(conn)); |
||||
return false; |
||||
} |
||||
if (r < STREAMING_HEADER_SIZE + 1) |
||||
{ |
||||
fprintf(stderr, _("%s: streaming header too small: %i\n"), |
||||
progname, r); |
||||
return false; |
||||
} |
||||
if (copybuf[0] != 'w') |
||||
{ |
||||
fprintf(stderr, _("%s: unrecognized streaming header: \"%c\"\n"), |
||||
progname, copybuf[0]); |
||||
return false; |
||||
} |
||||
|
||||
/* Extract WAL location for this block */ |
||||
memcpy(&blockpos, copybuf + 1, 8); |
||||
xlogoff = blockpos.xrecoff % XLOG_SEG_SIZE; |
||||
|
||||
/*
|
||||
* Verify that the initial location in the stream matches where we |
||||
* think we are. |
||||
*/ |
||||
if (walfile == -1) |
||||
{ |
||||
/* No file open yet */ |
||||
if (xlogoff != 0) |
||||
{ |
||||
fprintf(stderr, _("%s: received xlog record for offset %u with no file open\n"), |
||||
progname, xlogoff); |
||||
return false; |
||||
} |
||||
} |
||||
else |
||||
{ |
||||
/* More data in existing segment */ |
||||
/* XXX: store seek value don't reseek all the time */ |
||||
if (lseek(walfile, 0, SEEK_CUR) != xlogoff) |
||||
{ |
||||
fprintf(stderr, _("%s: got WAL data offset %08x, expected %08x\n"), |
||||
progname, xlogoff, (int) lseek(walfile, 0, SEEK_CUR)); |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
bytes_left = r - STREAMING_HEADER_SIZE; |
||||
bytes_written = 0; |
||||
|
||||
while (bytes_left) |
||||
{ |
||||
int bytes_to_write; |
||||
|
||||
/*
|
||||
* If crossing a WAL boundary, only write up until we reach |
||||
* XLOG_SEG_SIZE. |
||||
*/ |
||||
if (xlogoff + bytes_left > XLOG_SEG_SIZE) |
||||
bytes_to_write = XLOG_SEG_SIZE - xlogoff; |
||||
else |
||||
bytes_to_write = bytes_left; |
||||
|
||||
if (walfile == -1) |
||||
{ |
||||
walfile = open_walfile(blockpos, timeline, |
||||
basedir, current_walfile_name); |
||||
if (walfile == -1) |
||||
/* Error logged by open_walfile */ |
||||
return false; |
||||
} |
||||
|
||||
if (write(walfile, |
||||
copybuf + STREAMING_HEADER_SIZE + bytes_written, |
||||
bytes_to_write) != bytes_to_write) |
||||
{ |
||||
fprintf(stderr, _("%s: could not write %u bytes to WAL file %s: %s\n"), |
||||
progname, |
||||
bytes_to_write, |
||||
current_walfile_name, |
||||
strerror(errno)); |
||||
return false; |
||||
} |
||||
|
||||
/* Write was successful, advance our position */ |
||||
bytes_written += bytes_to_write; |
||||
bytes_left -= bytes_to_write; |
||||
XLByteAdvance(blockpos, bytes_to_write); |
||||
xlogoff += bytes_to_write; |
||||
|
||||
/* Did we reach the end of a WAL segment? */ |
||||
if (blockpos.xrecoff % XLOG_SEG_SIZE == 0) |
||||
{ |
||||
fsync(walfile); |
||||
close(walfile); |
||||
walfile = -1; |
||||
xlogoff = 0; |
||||
|
||||
if (segment_finish != NULL) |
||||
{ |
||||
/*
|
||||
* Callback when the segment finished, and return if it |
||||
* told us to. |
||||
*/ |
||||
if (segment_finish(blockpos, timeline)) |
||||
return true; |
||||
} |
||||
} |
||||
} |
||||
/* No more data left to write, start receiving next copy packet */ |
||||
} |
||||
|
||||
/*
|
||||
* The only way to get out of the loop is if the server shut down the |
||||
* replication stream. If it's a controlled shutdown, the server will send |
||||
* a shutdown message, and we'll return the latest xlog location that has |
||||
* been streamed. |
||||
*/ |
||||
|
||||
res = PQgetResult(conn); |
||||
if (PQresultStatus(res) != PGRES_COMMAND_OK) |
||||
{ |
||||
fprintf(stderr, _("%s: unexpected termination of replication stream: %s\n"), |
||||
progname, PQresultErrorMessage(res)); |
||||
return false; |
||||
} |
||||
PQclear(res); |
||||
return true; |
||||
} |
||||
@ -0,0 +1,22 @@ |
||||
#include "access/xlogdefs.h" |
||||
|
||||
/*
|
||||
* Called whenever a segment is finished, return true to stop |
||||
* the streaming at this point. |
||||
*/ |
||||
typedef bool (*segment_finish_callback)(XLogRecPtr segendpos, uint32 timeline); |
||||
|
||||
/*
|
||||
* Called before trying to read more data. Return true to stop |
||||
* the streaming at this point. |
||||
*/ |
||||
typedef bool (*stream_continue_callback)(void); |
||||
|
||||
extern bool ReceiveXlogStream(PGconn *conn, |
||||
XLogRecPtr startpos, |
||||
uint32 timeline, |
||||
char *sysidentifier, |
||||
char *basedir, |
||||
segment_finish_callback segment_finish, |
||||
stream_continue_callback stream_continue, |
||||
int standby_message_timeout); |
||||
@ -0,0 +1,165 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* streamutil.c - utility functions for pg_basebackup and pg_receivelog |
||||
* |
||||
* Author: Magnus Hagander <magnus@hagander.net> |
||||
* |
||||
* Portions Copyright (c) 1996-2011, PostgreSQL Global Development Group |
||||
* |
||||
* IDENTIFICATION |
||||
* src/bin/pg_basebackup/streamutil.c |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
|
||||
/*
|
||||
* We have to use postgres.h not postgres_fe.h here, because there's so much |
||||
* backend-only stuff in the XLOG include files we need. But we need a |
||||
* frontend-ish environment otherwise. Hence this ugly hack. |
||||
*/ |
||||
#define FRONTEND 1 |
||||
#include "postgres.h" |
||||
#include "streamutil.h" |
||||
|
||||
#include <stdio.h> |
||||
#include <string.h> |
||||
|
||||
const char *progname; |
||||
char *dbhost = NULL; |
||||
char *dbuser = NULL; |
||||
char *dbport = NULL; |
||||
int dbgetpassword = 0; /* 0=auto, -1=never, 1=always */ |
||||
static char *dbpassword = NULL; |
||||
PGconn *conn = NULL; |
||||
|
||||
/*
|
||||
* strdup() and malloc() replacements that prints an error and exits |
||||
* if something goes wrong. Can never return NULL. |
||||
*/ |
||||
char * |
||||
xstrdup(const char *s) |
||||
{ |
||||
char *result; |
||||
|
||||
result = strdup(s); |
||||
if (!result) |
||||
{ |
||||
fprintf(stderr, _("%s: out of memory\n"), progname); |
||||
exit(1); |
||||
} |
||||
return result; |
||||
} |
||||
|
||||
void * |
||||
xmalloc0(int size) |
||||
{ |
||||
void *result; |
||||
|
||||
result = malloc(size); |
||||
if (!result) |
||||
{ |
||||
fprintf(stderr, _("%s: out of memory\n"), progname); |
||||
exit(1); |
||||
} |
||||
MemSet(result, 0, size); |
||||
return result; |
||||
} |
||||
|
||||
|
||||
PGconn * |
||||
GetConnection(void) |
||||
{ |
||||
PGconn *tmpconn; |
||||
int argcount = 4; /* dbname, replication, fallback_app_name,
|
||||
* password */ |
||||
int i; |
||||
const char **keywords; |
||||
const char **values; |
||||
char *password = NULL; |
||||
|
||||
if (dbhost) |
||||
argcount++; |
||||
if (dbuser) |
||||
argcount++; |
||||
if (dbport) |
||||
argcount++; |
||||
|
||||
keywords = xmalloc0((argcount + 1) * sizeof(*keywords)); |
||||
values = xmalloc0((argcount + 1) * sizeof(*values)); |
||||
|
||||
keywords[0] = "dbname"; |
||||
values[0] = "replication"; |
||||
keywords[1] = "replication"; |
||||
values[1] = "true"; |
||||
keywords[2] = "fallback_application_name"; |
||||
values[2] = progname; |
||||
i = 3; |
||||
if (dbhost) |
||||
{ |
||||
keywords[i] = "host"; |
||||
values[i] = dbhost; |
||||
i++; |
||||
} |
||||
if (dbuser) |
||||
{ |
||||
keywords[i] = "user"; |
||||
values[i] = dbuser; |
||||
i++; |
||||
} |
||||
if (dbport) |
||||
{ |
||||
keywords[i] = "port"; |
||||
values[i] = dbport; |
||||
i++; |
||||
} |
||||
|
||||
while (true) |
||||
{ |
||||
if (password) |
||||
free(password); |
||||
|
||||
if (dbpassword) |
||||
{ |
||||
/*
|
||||
* We've saved a password when a previous connection succeeded, |
||||
* meaning this is the call for a second session to the same |
||||
* database, so just forcibly reuse that password. |
||||
*/ |
||||
keywords[argcount - 1] = "password"; |
||||
values[argcount - 1] = dbpassword; |
||||
dbgetpassword = -1; /* Don't try again if this fails */ |
||||
} |
||||
else if (dbgetpassword == 1) |
||||
{ |
||||
password = simple_prompt(_("Password: "), 100, false); |
||||
keywords[argcount - 1] = "password"; |
||||
values[argcount - 1] = password; |
||||
} |
||||
|
||||
tmpconn = PQconnectdbParams(keywords, values, true); |
||||
|
||||
if (PQstatus(tmpconn) == CONNECTION_BAD && |
||||
PQconnectionNeedsPassword(tmpconn) && |
||||
dbgetpassword != -1) |
||||
{ |
||||
dbgetpassword = 1; /* ask for password next time */ |
||||
PQfinish(tmpconn); |
||||
continue; |
||||
} |
||||
|
||||
if (PQstatus(tmpconn) != CONNECTION_OK) |
||||
{ |
||||
fprintf(stderr, _("%s: could not connect to server: %s\n"), |
||||
progname, PQerrorMessage(tmpconn)); |
||||
exit(1); |
||||
} |
||||
|
||||
/* Connection ok! */ |
||||
free(values); |
||||
free(keywords); |
||||
|
||||
/* Store the password for next run */ |
||||
if (password) |
||||
dbpassword = password; |
||||
return tmpconn; |
||||
} |
||||
} |
||||
@ -0,0 +1,22 @@ |
||||
#include "libpq-fe.h" |
||||
|
||||
extern const char *progname; |
||||
extern char *dbhost; |
||||
extern char *dbuser; |
||||
extern char *dbport; |
||||
extern int dbgetpassword; |
||||
|
||||
/* Connection kept global so we can disconnect easily */ |
||||
extern PGconn *conn; |
||||
|
||||
#define disconnect_and_exit(code) \ |
||||
{ \
|
||||
if (conn != NULL) PQfinish(conn); \
|
||||
exit(code); \
|
||||
} |
||||
|
||||
|
||||
char *xstrdup(const char *s); |
||||
void *xmalloc0(int size); |
||||
|
||||
PGconn *GetConnection(void); |
||||
Loading…
Reference in new issue