@ -35,6 +35,7 @@
# include "compress_io.h"
# include "dumputils.h"
# include "parallel.h"
# include <dirent.h>
# include <sys/stat.h>
@ -50,6 +51,7 @@ typedef struct
cfp * dataFH ; /* currently open data file */
cfp * blobsTocFH ; /* file handle for blobs.toc */
ParallelState * pstate ; /* for parallel backup / restore */
} lclContext ;
typedef struct
@ -70,6 +72,7 @@ static int _ReadByte(ArchiveHandle *);
static size_t _WriteBuf ( ArchiveHandle * AH , const void * buf , size_t len ) ;
static size_t _ReadBuf ( ArchiveHandle * AH , void * buf , size_t len ) ;
static void _CloseArchive ( ArchiveHandle * AH ) ;
static void _ReopenArchive ( ArchiveHandle * AH ) ;
static void _PrintTocData ( ArchiveHandle * AH , TocEntry * te , RestoreOptions * ropt ) ;
static void _WriteExtraToc ( ArchiveHandle * AH , TocEntry * te ) ;
@ -82,8 +85,17 @@ static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
static void _EndBlobs ( ArchiveHandle * AH , TocEntry * te ) ;
static void _LoadBlobs ( ArchiveHandle * AH , RestoreOptions * ropt ) ;
static char * prependDirectory ( ArchiveHandle * AH , const char * relativeFilename ) ;
static void _Clone ( ArchiveHandle * AH ) ;
static void _DeClone ( ArchiveHandle * AH ) ;
static char * _MasterStartParallelItem ( ArchiveHandle * AH , TocEntry * te , T_Action act ) ;
static int _MasterEndParallelItem ( ArchiveHandle * AH , TocEntry * te ,
const char * str , T_Action act ) ;
static char * _WorkerJobRestoreDirectory ( ArchiveHandle * AH , TocEntry * te ) ;
static char * _WorkerJobDumpDirectory ( ArchiveHandle * AH , TocEntry * te ) ;
static void setFilePath ( ArchiveHandle * AH , char * buf ,
const char * relativeFilename ) ;
/*
* Init routine required by ALL formats . This is a global routine
@ -110,7 +122,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
AH - > WriteBufPtr = _WriteBuf ;
AH - > ReadBufPtr = _ReadBuf ;
AH - > ClosePtr = _CloseArchive ;
AH - > ReopenPtr = NULL ;
AH - > ReopenPtr = _ReopenArchive ;
AH - > PrintTocDataPtr = _PrintTocData ;
AH - > ReadExtraTocPtr = _ReadExtraToc ;
AH - > WriteExtraTocPtr = _WriteExtraToc ;
@ -121,8 +133,14 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
AH - > EndBlobPtr = _EndBlob ;
AH - > EndBlobsPtr = _EndBlobs ;
AH - > ClonePtr = NULL ;
AH - > DeClonePtr = NULL ;
AH - > ClonePtr = _Clone ;
AH - > DeClonePtr = _DeClone ;
AH - > WorkerJobRestorePtr = _WorkerJobRestoreDirectory ;
AH - > WorkerJobDumpPtr = _WorkerJobDumpDirectory ;
AH - > MasterStartParallelItemPtr = _MasterStartParallelItem ;
AH - > MasterEndParallelItemPtr = _MasterEndParallelItem ;
/* Set up our private context */
ctx = ( lclContext * ) pg_malloc0 ( sizeof ( lclContext ) ) ;
@ -146,16 +164,41 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
if ( AH - > mode = = archModeWrite )
{
if ( mkdir ( ctx - > directory , 0700 ) < 0 )
struct stat st ;
bool is_empty = false ;
/* we accept an empty existing directory */
if ( stat ( ctx - > directory , & st ) = = 0 & & S_ISDIR ( st . st_mode ) )
{
DIR * dir = opendir ( ctx - > directory ) ;
if ( dir )
{
struct dirent * d ;
is_empty = true ;
while ( ( d = readdir ( dir ) ) )
{
if ( strcmp ( d - > d_name , " . " ) ! = 0 & & strcmp ( d - > d_name , " .. " ) ! = 0 )
{
is_empty = false ;
break ;
}
}
closedir ( dir ) ;
}
}
if ( ! is_empty & & mkdir ( ctx - > directory , 0700 ) < 0 )
exit_horribly ( modulename , " could not create directory \" %s \" : %s \n " ,
ctx - > directory , strerror ( errno ) ) ;
}
else
{ /* Read Mode */
char * fname ;
char fname [ MAXPGPATH ] ;
cfp * tocFH ;
fname = prependDirectory ( AH , " toc.dat " ) ;
setFilePath ( AH , fname , " toc.dat " ) ;
tocFH = cfopen_read ( fname , PG_BINARY_R ) ;
if ( tocFH = = NULL )
@ -281,9 +324,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
{
lclTocEntry * tctx = ( lclTocEntry * ) te - > formatData ;
lclContext * ctx = ( lclContext * ) AH - > formatData ;
char * fname ;
char fname [ MAXPGPATH ] ;
fname = prependDirectory ( AH , tctx - > filename ) ;
setFilePath ( AH , fname , tctx - > filename ) ;
ctx - > dataFH = cfopen_write ( fname , PG_BINARY_W , AH - > compression ) ;
if ( ctx - > dataFH = = NULL )
@ -308,6 +351,9 @@ _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
if ( dLen = = 0 )
return 0 ;
/* Are we aborting? */
checkAborting ( AH ) ;
return cfwrite ( data , dLen , ctx - > dataFH ) ;
}
@ -375,8 +421,9 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt)
_LoadBlobs ( AH , ropt ) ;
else
{
char * fname = prependDirectory ( AH , tctx - > filename ) ;
char fname [ MAXPGPATH ] ;
setFilePath ( AH , fname , tctx - > filename ) ;
_PrintFileData ( AH , fname , ropt ) ;
}
}
@ -386,12 +433,12 @@ _LoadBlobs(ArchiveHandle *AH, RestoreOptions *ropt)
{
Oid oid ;
lclContext * ctx = ( lclContext * ) AH - > formatData ;
char * fname ;
char fname [ MAXPGPATH ] ;
char line [ MAXPGPATH ] ;
StartRestoreBlobs ( AH ) ;
fname = prependDirectory ( AH , " blobs.toc " ) ;
setFilePath ( AH , fname , " blobs.toc " ) ;
ctx - > blobsTocFH = cfopen_read ( fname , PG_BINARY_R ) ;
@ -474,6 +521,9 @@ _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
lclContext * ctx = ( lclContext * ) AH - > formatData ;
size_t res ;
/* Are we aborting? */
checkAborting ( AH ) ;
res = cfwrite ( buf , len , ctx - > dataFH ) ;
if ( res ! = len )
exit_horribly ( modulename , " could not write to output file: %s \n " ,
@ -518,7 +568,12 @@ _CloseArchive(ArchiveHandle *AH)
if ( AH - > mode = = archModeWrite )
{
cfp * tocFH ;
char * fname = prependDirectory ( AH , " toc.dat " ) ;
char fname [ MAXPGPATH ] ;
setFilePath ( AH , fname , " toc.dat " ) ;
/* this will actually fork the processes for a parallel backup */
ctx - > pstate = ParallelBackupStart ( AH , NULL ) ;
/* The TOC is always created uncompressed */
tocFH = cfopen_write ( fname , PG_BINARY_W , 0 ) ;
@ -539,11 +594,25 @@ _CloseArchive(ArchiveHandle *AH)
if ( cfclose ( tocFH ) ! = 0 )
exit_horribly ( modulename , " could not close TOC file: %s \n " ,
strerror ( errno ) ) ;
WriteDataChunks ( AH ) ;
WriteDataChunks ( AH , ctx - > pstate ) ;
ParallelBackupEnd ( AH , ctx - > pstate ) ;
}
AH - > FH = NULL ;
}
/*
* Reopen the archive ' s file handle .
*/
static void
_ReopenArchive ( ArchiveHandle * AH )
{
/*
* Our TOC is in memory , our data files are opened by each child anyway as
* they are separate . We support reopening the archive by just doing
* nothing .
*/
}
/*
* BLOB support
@ -560,9 +629,9 @@ static void
_StartBlobs ( ArchiveHandle * AH , TocEntry * te )
{
lclContext * ctx = ( lclContext * ) AH - > formatData ;
char * fname ;
char fname [ MAXPGPATH ] ;
fname = prependDirectory ( AH , " blobs.toc " ) ;
setFilePath ( AH , fname , " blobs.toc " ) ;
/* The blob TOC file is never compressed */
ctx - > blobsTocFH = cfopen_write ( fname , " ab " , 0 ) ;
@ -627,12 +696,16 @@ _EndBlobs(ArchiveHandle *AH, TocEntry *te)
ctx - > blobsTocFH = NULL ;
}
static char *
prependDirectory ( ArchiveHandle * AH , const char * relativeFilename )
/*
* Gets a relative file name and prepends the output directory , writing the
* result to buf . The caller needs to make sure that buf is MAXPGPATH bytes
* big . Can ' t use a static char [ MAXPGPATH ] inside the function because we run
* multithreaded on Windows .
*/
static void
setFilePath ( ArchiveHandle * AH , char * buf , const char * relativeFilename )
{
lclContext * ctx = ( lclContext * ) AH - > formatData ;
static char buf [ MAXPGPATH ] ;
char * dname ;
dname = ctx - > directory ;
@ -643,6 +716,157 @@ prependDirectory(ArchiveHandle *AH, const char *relativeFilename)
strcpy ( buf , dname ) ;
strcat ( buf , " / " ) ;
strcat ( buf , relativeFilename ) ;
}
/*
* Clone format - specific fields during parallel restoration .
*/
static void
_Clone ( ArchiveHandle * AH )
{
lclContext * ctx = ( lclContext * ) AH - > formatData ;
AH - > formatData = ( lclContext * ) pg_malloc ( sizeof ( lclContext ) ) ;
memcpy ( AH - > formatData , ctx , sizeof ( lclContext ) ) ;
ctx = ( lclContext * ) AH - > formatData ;
/*
* Note : we do not make a local lo_buf because we expect at most one BLOBS
* entry per archive , so no parallelism is possible . Likewise ,
* TOC - entry - local state isn ' t an issue because any one TOC entry is
* touched by just one worker child .
*/
/*
* We also don ' t copy the ParallelState pointer ( pstate ) , only the master
* process ever writes to it .
*/
}
static void
_DeClone ( ArchiveHandle * AH )
{
lclContext * ctx = ( lclContext * ) AH - > formatData ;
free ( ctx ) ;
}
/*
* This function is executed in the parent process . Depending on the desired
* action ( dump or restore ) it creates a string that is understood by the
* _WorkerJobDump / _WorkerJobRestore functions of the dump format .
*/
static char *
_MasterStartParallelItem ( ArchiveHandle * AH , TocEntry * te , T_Action act )
{
/*
* A static char is okay here , even on Windows because we call this
* function only from one process ( the master ) .
*/
static char buf [ 64 ] ;
if ( act = = ACT_DUMP )
snprintf ( buf , sizeof ( buf ) , " DUMP %d " , te - > dumpId ) ;
else if ( act = = ACT_RESTORE )
snprintf ( buf , sizeof ( buf ) , " RESTORE %d " , te - > dumpId ) ;
return buf ;
}
/*
* This function is executed in the child of a parallel backup for the
* directory archive and dumps the actual data .
*
* We are currently returning only the DumpId so theoretically we could
* make this function returning an int ( or a DumpId ) . However , to
* facilitate further enhancements and because sooner or later we need to
* convert this to a string and send it via a message anyway , we stick with
* char * . It is parsed on the other side by the _EndMasterParallel ( )
* function of the respective dump format .
*/
static char *
_WorkerJobDumpDirectory ( ArchiveHandle * AH , TocEntry * te )
{
/*
* short fixed - size string + some ID so far , this needs to be malloc ' ed
* instead of static because we work with threads on windows
*/
const int buflen = 64 ;
char * buf = ( char * ) pg_malloc ( buflen ) ;
lclTocEntry * tctx = ( lclTocEntry * ) te - > formatData ;
/* This should never happen */
if ( ! tctx )
exit_horribly ( modulename , " Error during backup \n " ) ;
/*
* This function returns void . We either fail and die horribly or
* succeed . . . A failure will be detected by the parent when the child dies
* unexpectedly .
*/
WriteDataChunksForTocEntry ( AH , te ) ;
snprintf ( buf , buflen , " OK DUMP %d " , te - > dumpId ) ;
return buf ;
}
/*
* This function is executed in the child of a parallel backup for the
* directory archive and dumps the actual data .
*/
static char *
_WorkerJobRestoreDirectory ( ArchiveHandle * AH , TocEntry * te )
{
/*
* short fixed - size string + some ID so far , this needs to be malloc ' ed
* instead of static because we work with threads on windows
*/
const int buflen = 64 ;
char * buf = ( char * ) pg_malloc ( buflen ) ;
ParallelArgs pargs ;
int status ;
pargs . AH = AH ;
pargs . te = te ;
status = parallel_restore ( & pargs ) ;
snprintf ( buf , buflen , " OK RESTORE %d %d %d " , te - > dumpId , status ,
status = = WORKER_IGNORED_ERRORS ? AH - > public . n_errors : 0 ) ;
return buf ;
}
/*
* This function is executed in the parent process . It analyzes the response of
* the _WorkerJobDumpDirectory / _WorkerJobRestoreDirectory functions of the
* respective dump format .
*/
static int
_MasterEndParallelItem ( ArchiveHandle * AH , TocEntry * te , const char * str , T_Action act )
{
DumpId dumpId ;
int nBytes ,
n_errors ;
int status = 0 ;
if ( act = = ACT_DUMP )
{
sscanf ( str , " %u%n " , & dumpId , & nBytes ) ;
Assert ( dumpId = = te - > dumpId ) ;
Assert ( nBytes = = strlen ( str ) ) ;
}
else if ( act = = ACT_RESTORE )
{
sscanf ( str , " %u %u %u%n " , & dumpId , & status , & n_errors , & nBytes ) ;
Assert ( dumpId = = te - > dumpId ) ;
Assert ( nBytes = = strlen ( str ) ) ;
AH - > public . n_errors + = n_errors ;
}
return status ;
}
Andrew Dunstan
12 years ago