mirror of https://github.com/postgres/postgres
basebackup.c is already a pretty big and complicated file, so it makes more sense to keep the backup manifest support routines in a separate file, for clarity and ease of maintenance. Discussion: http://postgr.es/m/CA+TgmoavRak5OdP76P8eJExDYhPEKWjMb0sxW7dF01dWFgE=uA@mail.gmail.compull/54/head
Robert Haas
5 years ago
parent
1e324cb0e7
commit
079ac29d4d
@ -0,0 +1,375 @@ |
|||||||
|
/*-------------------------------------------------------------------------
|
||||||
|
* |
||||||
|
* backup_manifest.c |
||||||
|
* code for generating and sending a backup manifest |
||||||
|
* |
||||||
|
* Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group |
||||||
|
* |
||||||
|
* IDENTIFICATION |
||||||
|
* src/backend/replication/backup_manifest.c |
||||||
|
* |
||||||
|
*------------------------------------------------------------------------- |
||||||
|
*/ |
||||||
|
#include "postgres.h" |
||||||
|
|
||||||
|
#include "access/timeline.h" |
||||||
|
#include "libpq/libpq.h" |
||||||
|
#include "libpq/pqformat.h" |
||||||
|
#include "mb/pg_wchar.h" |
||||||
|
#include "replication/backup_manifest.h" |
||||||
|
#include "utils/builtins.h" |
||||||
|
#include "utils/json.h" |
||||||
|
|
||||||
|
/*
|
||||||
|
* Does the user want a backup manifest? |
||||||
|
* |
||||||
|
* It's simplest to always have a manifest_info object, so that we don't need |
||||||
|
* checks for NULL pointers in too many places. However, if the user doesn't |
||||||
|
* want a manifest, we set manifest->buffile to NULL. |
||||||
|
*/ |
||||||
|
static inline bool |
||||||
|
IsManifestEnabled(manifest_info *manifest) |
||||||
|
{ |
||||||
|
return (manifest->buffile != NULL); |
||||||
|
} |
||||||
|
|
||||||
|
/*
|
||||||
|
* Convenience macro for appending data to the backup manifest. |
||||||
|
*/ |
||||||
|
#define AppendToManifest(manifest, ...) \ |
||||||
|
{ \
|
||||||
|
char *_manifest_s = psprintf(__VA_ARGS__); \
|
||||||
|
AppendStringToManifest(manifest, _manifest_s); \
|
||||||
|
pfree(_manifest_s); \
|
||||||
|
} |
||||||
|
|
||||||
|
/*
|
||||||
|
* Initialize state so that we can construct a backup manifest. |
||||||
|
* |
||||||
|
* NB: Although the checksum type for the data files is configurable, the |
||||||
|
* checksum for the manifest itself always uses SHA-256. See comments in |
||||||
|
* SendBackupManifest. |
||||||
|
*/ |
||||||
|
void |
||||||
|
InitializeManifest(manifest_info *manifest, manifest_option want_manifest, |
||||||
|
pg_checksum_type manifest_checksum_type) |
||||||
|
{ |
||||||
|
if (want_manifest == MANIFEST_OPTION_NO) |
||||||
|
manifest->buffile = NULL; |
||||||
|
else |
||||||
|
manifest->buffile = BufFileCreateTemp(false); |
||||||
|
manifest->checksum_type = manifest_checksum_type; |
||||||
|
pg_sha256_init(&manifest->manifest_ctx); |
||||||
|
manifest->manifest_size = UINT64CONST(0); |
||||||
|
manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE); |
||||||
|
manifest->first_file = true; |
||||||
|
manifest->still_checksumming = true; |
||||||
|
|
||||||
|
if (want_manifest != MANIFEST_OPTION_NO) |
||||||
|
AppendToManifest(manifest, |
||||||
|
"{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n" |
||||||
|
"\"Files\": ["); |
||||||
|
} |
||||||
|
|
||||||
|
/*
|
||||||
|
* Append a cstring to the manifest. |
||||||
|
*/ |
||||||
|
void |
||||||
|
AppendStringToManifest(manifest_info *manifest, char *s) |
||||||
|
{ |
||||||
|
int len = strlen(s); |
||||||
|
size_t written; |
||||||
|
|
||||||
|
Assert(manifest != NULL); |
||||||
|
if (manifest->still_checksumming) |
||||||
|
pg_sha256_update(&manifest->manifest_ctx, (uint8 *) s, len); |
||||||
|
written = BufFileWrite(manifest->buffile, s, len); |
||||||
|
if (written != len) |
||||||
|
ereport(ERROR, |
||||||
|
(errcode_for_file_access(), |
||||||
|
errmsg("could not write to temporary file: %m"))); |
||||||
|
manifest->manifest_size += len; |
||||||
|
} |
||||||
|
|
||||||
|
/*
|
||||||
|
* Add an entry to the backup manifest for a file. |
||||||
|
*/ |
||||||
|
void |
||||||
|
AddFileToManifest(manifest_info *manifest, const char *spcoid, |
||||||
|
const char *pathname, size_t size, pg_time_t mtime, |
||||||
|
pg_checksum_context *checksum_ctx) |
||||||
|
{ |
||||||
|
char pathbuf[MAXPGPATH]; |
||||||
|
int pathlen; |
||||||
|
StringInfoData buf; |
||||||
|
|
||||||
|
if (!IsManifestEnabled(manifest)) |
||||||
|
return; |
||||||
|
|
||||||
|
/*
|
||||||
|
* If this file is part of a tablespace, the pathname passed to this |
||||||
|
* function will be relative to the tar file that contains it. We want the |
||||||
|
* pathname relative to the data directory (ignoring the intermediate |
||||||
|
* symlink traversal). |
||||||
|
*/ |
||||||
|
if (spcoid != NULL) |
||||||
|
{ |
||||||
|
snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", spcoid, |
||||||
|
pathname); |
||||||
|
pathname = pathbuf; |
||||||
|
} |
||||||
|
|
||||||
|
/*
|
||||||
|
* Each file's entry needs to be separated from any entry that follows by a |
||||||
|
* comma, but there's no comma before the first one or after the last one. |
||||||
|
* To make that work, adding a file to the manifest starts by terminating |
||||||
|
* the most recently added line, with a comma if appropriate, but does not |
||||||
|
* terminate the line inserted for this file. |
||||||
|
*/ |
||||||
|
initStringInfo(&buf); |
||||||
|
if (manifest->first_file) |
||||||
|
{ |
||||||
|
appendStringInfoString(&buf, "\n"); |
||||||
|
manifest->first_file = false; |
||||||
|
} |
||||||
|
else |
||||||
|
appendStringInfoString(&buf, ",\n"); |
||||||
|
|
||||||
|
/*
|
||||||
|
* Write the relative pathname to this file out to the manifest. The |
||||||
|
* manifest is always stored in UTF-8, so we have to encode paths that are |
||||||
|
* not valid in that encoding. |
||||||
|
*/ |
||||||
|
pathlen = strlen(pathname); |
||||||
|
if (!manifest->force_encode && |
||||||
|
pg_verify_mbstr(PG_UTF8, pathname, pathlen, true)) |
||||||
|
{ |
||||||
|
appendStringInfoString(&buf, "{ \"Path\": "); |
||||||
|
escape_json(&buf, pathname); |
||||||
|
appendStringInfoString(&buf, ", "); |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
appendStringInfoString(&buf, "{ \"Encoded-Path\": \""); |
||||||
|
enlargeStringInfo(&buf, 2 * pathlen); |
||||||
|
buf.len += hex_encode((char *) pathname, pathlen, |
||||||
|
&buf.data[buf.len]); |
||||||
|
appendStringInfoString(&buf, "\", "); |
||||||
|
} |
||||||
|
|
||||||
|
appendStringInfo(&buf, "\"Size\": %zu, ", size); |
||||||
|
|
||||||
|
/*
|
||||||
|
* Convert last modification time to a string and append it to the |
||||||
|
* manifest. Since it's not clear what time zone to use and since time |
||||||
|
* zone definitions can change, possibly causing confusion, use GMT |
||||||
|
* always. |
||||||
|
*/ |
||||||
|
appendStringInfoString(&buf, "\"Last-Modified\": \""); |
||||||
|
enlargeStringInfo(&buf, 128); |
||||||
|
buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z", |
||||||
|
pg_gmtime(&mtime)); |
||||||
|
appendStringInfoString(&buf, "\""); |
||||||
|
|
||||||
|
/* Add checksum information. */ |
||||||
|
if (checksum_ctx->type != CHECKSUM_TYPE_NONE) |
||||||
|
{ |
||||||
|
uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH]; |
||||||
|
int checksumlen; |
||||||
|
|
||||||
|
checksumlen = pg_checksum_final(checksum_ctx, checksumbuf); |
||||||
|
|
||||||
|
appendStringInfo(&buf, |
||||||
|
", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"", |
||||||
|
pg_checksum_type_name(checksum_ctx->type)); |
||||||
|
enlargeStringInfo(&buf, 2 * checksumlen); |
||||||
|
buf.len += hex_encode((char *) checksumbuf, checksumlen, |
||||||
|
&buf.data[buf.len]); |
||||||
|
appendStringInfoString(&buf, "\""); |
||||||
|
} |
||||||
|
|
||||||
|
/* Close out the object. */ |
||||||
|
appendStringInfoString(&buf, " }"); |
||||||
|
|
||||||
|
/* OK, add it to the manifest. */ |
||||||
|
AppendStringToManifest(manifest, buf.data); |
||||||
|
|
||||||
|
/* Avoid leaking memory. */ |
||||||
|
pfree(buf.data); |
||||||
|
} |
||||||
|
|
||||||
|
/*
|
||||||
|
* Add information about the WAL that will need to be replayed when restoring |
||||||
|
* this backup to the manifest. |
||||||
|
*/ |
||||||
|
void |
||||||
|
AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr, |
||||||
|
TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli) |
||||||
|
{ |
||||||
|
List *timelines; |
||||||
|
ListCell *lc; |
||||||
|
bool first_wal_range = true; |
||||||
|
bool found_start_timeline = false; |
||||||
|
|
||||||
|
if (!IsManifestEnabled(manifest)) |
||||||
|
return; |
||||||
|
|
||||||
|
/* Terminate the list of files. */ |
||||||
|
AppendStringToManifest(manifest, "\n],\n"); |
||||||
|
|
||||||
|
/* Read the timeline history for the ending timeline. */ |
||||||
|
timelines = readTimeLineHistory(endtli); |
||||||
|
|
||||||
|
/* Start a list of LSN ranges. */ |
||||||
|
AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n"); |
||||||
|
|
||||||
|
foreach(lc, timelines) |
||||||
|
{ |
||||||
|
TimeLineHistoryEntry *entry = lfirst(lc); |
||||||
|
XLogRecPtr tl_beginptr; |
||||||
|
|
||||||
|
/*
|
||||||
|
* We only care about timelines that were active during the backup. |
||||||
|
* Skip any that ended before the backup started. (Note that if |
||||||
|
* entry->end is InvalidXLogRecPtr, it means that the timeline has not |
||||||
|
* yet ended.) |
||||||
|
*/ |
||||||
|
if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr) |
||||||
|
continue; |
||||||
|
|
||||||
|
/*
|
||||||
|
* Because the timeline history file lists newer timelines before |
||||||
|
* older ones, the first timeline we encounter that is new enough to |
||||||
|
* matter ought to match the ending timeline of the backup. |
||||||
|
*/ |
||||||
|
if (first_wal_range && endtli != entry->tli) |
||||||
|
ereport(ERROR, |
||||||
|
errmsg("expected end timeline %u but found timeline %u", |
||||||
|
starttli, entry->tli)); |
||||||
|
|
||||||
|
if (!XLogRecPtrIsInvalid(entry->begin)) |
||||||
|
tl_beginptr = entry->begin; |
||||||
|
else |
||||||
|
{ |
||||||
|
tl_beginptr = startptr; |
||||||
|
|
||||||
|
/*
|
||||||
|
* If we reach a TLI that has no valid beginning LSN, there can't |
||||||
|
* be any more timelines in the history after this point, so we'd |
||||||
|
* better have arrived at the expected starting TLI. If not, |
||||||
|
* something's gone horribly wrong. |
||||||
|
*/ |
||||||
|
if (starttli != entry->tli) |
||||||
|
ereport(ERROR, |
||||||
|
errmsg("expected start timeline %u but found timeline %u", |
||||||
|
starttli, entry->tli)); |
||||||
|
} |
||||||
|
|
||||||
|
AppendToManifest(manifest, |
||||||
|
"%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }", |
||||||
|
first_wal_range ? "" : ",\n", |
||||||
|
entry->tli, |
||||||
|
(uint32) (tl_beginptr >> 32), (uint32) tl_beginptr, |
||||||
|
(uint32) (endptr >> 32), (uint32) endptr); |
||||||
|
|
||||||
|
if (starttli == entry->tli) |
||||||
|
{ |
||||||
|
found_start_timeline = true; |
||||||
|
break; |
||||||
|
} |
||||||
|
|
||||||
|
endptr = entry->begin; |
||||||
|
first_wal_range = false; |
||||||
|
} |
||||||
|
|
||||||
|
/*
|
||||||
|
* The last entry in the timeline history for the ending timeline should |
||||||
|
* be the ending timeline itself. Verify that this is what we observed. |
||||||
|
*/ |
||||||
|
if (!found_start_timeline) |
||||||
|
ereport(ERROR, |
||||||
|
errmsg("start timeline %u not found history of timeline %u", |
||||||
|
starttli, endtli)); |
||||||
|
|
||||||
|
/* Terminate the list of WAL ranges. */ |
||||||
|
AppendStringToManifest(manifest, "\n],\n"); |
||||||
|
} |
||||||
|
|
||||||
|
/*
|
||||||
|
* Finalize the backup manifest, and send it to the client. |
||||||
|
*/ |
||||||
|
void |
||||||
|
SendBackupManifest(manifest_info *manifest) |
||||||
|
{ |
||||||
|
StringInfoData protobuf; |
||||||
|
uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH]; |
||||||
|
char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH]; |
||||||
|
size_t manifest_bytes_done = 0; |
||||||
|
|
||||||
|
if (!IsManifestEnabled(manifest)) |
||||||
|
return; |
||||||
|
|
||||||
|
/*
|
||||||
|
* Append manifest checksum, so that the problems with the manifest itself |
||||||
|
* can be detected. |
||||||
|
* |
||||||
|
* We always use SHA-256 for this, regardless of what algorithm is chosen |
||||||
|
* for checksumming the files. If we ever want to make the checksum |
||||||
|
* algorithm used for the manifest file variable, the client will need a |
||||||
|
* way to figure out which algorithm to use as close to the beginning of |
||||||
|
* the manifest file as possible, to avoid having to read the whole thing |
||||||
|
* twice. |
||||||
|
*/ |
||||||
|
manifest->still_checksumming = false; |
||||||
|
pg_sha256_final(&manifest->manifest_ctx, checksumbuf); |
||||||
|
AppendStringToManifest(manifest, "\"Manifest-Checksum\": \""); |
||||||
|
hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf); |
||||||
|
checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0'; |
||||||
|
AppendStringToManifest(manifest, checksumstringbuf); |
||||||
|
AppendStringToManifest(manifest, "\"}\n"); |
||||||
|
|
||||||
|
/*
|
||||||
|
* We've written all the data to the manifest file. Rewind the file so |
||||||
|
* that we can read it all back. |
||||||
|
*/ |
||||||
|
if (BufFileSeek(manifest->buffile, 0, 0L, SEEK_SET)) |
||||||
|
ereport(ERROR, |
||||||
|
(errcode_for_file_access(), |
||||||
|
errmsg("could not rewind temporary file: %m"))); |
||||||
|
|
||||||
|
/* Send CopyOutResponse message */ |
||||||
|
pq_beginmessage(&protobuf, 'H'); |
||||||
|
pq_sendbyte(&protobuf, 0); /* overall format */ |
||||||
|
pq_sendint16(&protobuf, 0); /* natts */ |
||||||
|
pq_endmessage(&protobuf); |
||||||
|
|
||||||
|
/*
|
||||||
|
* Send CopyData messages. |
||||||
|
* |
||||||
|
* We choose to read back the data from the temporary file in chunks of |
||||||
|
* size BLCKSZ; this isn't necessary, but buffile.c uses that as the I/O |
||||||
|
* size, so it seems to make sense to match that value here. |
||||||
|
*/ |
||||||
|
while (manifest_bytes_done < manifest->manifest_size) |
||||||
|
{ |
||||||
|
char manifestbuf[BLCKSZ]; |
||||||
|
size_t bytes_to_read; |
||||||
|
size_t rc; |
||||||
|
|
||||||
|
bytes_to_read = Min(sizeof(manifestbuf), |
||||||
|
manifest->manifest_size - manifest_bytes_done); |
||||||
|
rc = BufFileRead(manifest->buffile, manifestbuf, bytes_to_read); |
||||||
|
if (rc != bytes_to_read) |
||||||
|
ereport(ERROR, |
||||||
|
(errcode_for_file_access(), |
||||||
|
errmsg("could not read from temporary file: %m"))); |
||||||
|
pq_putmessage('d', manifestbuf, bytes_to_read); |
||||||
|
manifest_bytes_done += bytes_to_read; |
||||||
|
} |
||||||
|
|
||||||
|
/* No more data, so send CopyDone message */ |
||||||
|
pq_putemptymessage('c'); |
||||||
|
|
||||||
|
/* Release resources */ |
||||||
|
BufFileClose(manifest->buffile); |
||||||
|
} |
||||||
@ -0,0 +1,51 @@ |
|||||||
|
/*-------------------------------------------------------------------------
|
||||||
|
* |
||||||
|
* backup_manifest.h |
||||||
|
* Routines for generating a backup manifest. |
||||||
|
* |
||||||
|
* Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group |
||||||
|
* |
||||||
|
* src/include/replication/backup_manifest.h |
||||||
|
* |
||||||
|
*------------------------------------------------------------------------- |
||||||
|
*/ |
||||||
|
#ifndef BACKUP_MANIFEST_H |
||||||
|
#define BACKUP_MANIFEST_H |
||||||
|
|
||||||
|
#include "access/xlogdefs.h" |
||||||
|
#include "common/checksum_helper.h" |
||||||
|
#include "pgtime.h" |
||||||
|
#include "storage/buffile.h" |
||||||
|
|
||||||
|
typedef enum manifest_option |
||||||
|
{ |
||||||
|
MANIFEST_OPTION_YES, |
||||||
|
MANIFEST_OPTION_NO, |
||||||
|
MANIFEST_OPTION_FORCE_ENCODE |
||||||
|
} manifest_option; |
||||||
|
|
||||||
|
typedef struct manifest_info |
||||||
|
{ |
||||||
|
BufFile *buffile; |
||||||
|
pg_checksum_type checksum_type; |
||||||
|
pg_sha256_ctx manifest_ctx; |
||||||
|
uint64 manifest_size; |
||||||
|
bool force_encode; |
||||||
|
bool first_file; |
||||||
|
bool still_checksumming; |
||||||
|
} manifest_info; |
||||||
|
|
||||||
|
extern void InitializeManifest(manifest_info *manifest, |
||||||
|
manifest_option want_manifest, |
||||||
|
pg_checksum_type manifest_checksum_type); |
||||||
|
extern void AppendStringToManifest(manifest_info *manifest, char *s); |
||||||
|
extern void AddFileToManifest(manifest_info *manifest, const char *spcoid, |
||||||
|
const char *pathname, size_t size, |
||||||
|
pg_time_t mtime, |
||||||
|
pg_checksum_context *checksum_ctx); |
||||||
|
extern void AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr, |
||||||
|
TimeLineID starttli, XLogRecPtr endptr, |
||||||
|
TimeLineID endtli); |
||||||
|
extern void SendBackupManifest(manifest_info *manifest); |
||||||
|
|
||||||
|
#endif |
||||||
Loading…
Reference in new issue