open-dsi
/
postgres
mirror of https://github.com/postgres/postgres
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

New two-stage sampling method for ANALYZE, as per discussions a few weeks

Browse Source 
ago.  This should give significantly better results when the density of
live tuples is not uniform throughout a table.  Manfred Koizar, with
minor kibitzing from Tom Lane.
REL8_0_STABLE
Tom Lane 22 years ago
parent 27edff700e
commit 9d6570b8a4
1 changed files with 226 additions and 163 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 389
      src/backend/commands/analyze.c

389
src/backend/commands/analyze.c
Unescape View File

@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/commands/analyze.c,v 1.71 2004/05/08 19:09:24 tgl Exp $
* $PostgreSQL: pgsql/src/backend/commands/analyze.c,v 1.72 2004/05/23 21:24:02 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@ -39,6 +39,16 @@
#include "utils/tuplesort.h"
/* Data structure for Algorithm S from Knuth 3.4.2 */
typedef struct
{
BlockNumber N; /* number of blocks, known in advance */
int n; /* desired sample size */
BlockNumber t; /* current block number */
int m; /* blocks selected so far */
} BlockSamplerData;
typedef BlockSamplerData *BlockSampler;
/* Per-index data for ANALYZE */
typedef struct AnlIndexData
{
@ -57,6 +67,10 @@ static int elevel = -1;
static MemoryContext anl_context = NULL;
static void BlockSampler_Init(BlockSampler bs, BlockNumber nblocks,
int samplesize);
static bool BlockSampler_HasMore(BlockSampler bs);
static BlockNumber BlockSampler_Next(BlockSampler bs);
static void compute_index_stats(Relation onerel, double totalrows,
AnlIndexData *indexdata, int nindexes,
HeapTuple *rows, int numrows,
@ -66,7 +80,7 @@ static int acquire_sample_rows(Relation onerel, HeapTuple *rows,
int targrows, double *totalrows);
static double random_fract(void);
static double init_selection_state(int n);
static double select_next_random_record(double t, int n, double *stateptr);
static double get_next_S(double t, int n, double *stateptr);
static int compare_rows(const void *a, const void *b);
static void update_attstats(Oid relid, int natts, VacAttrStats **vacattrstats);
static Datum std_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
@ -637,16 +651,118 @@ examine_attribute(Relation onerel, int attnum)
return stats;
}
/*
* BlockSampler_Init -- prepare for random sampling of blocknumbers
*
* BlockSampler is used for stage one of our new two-stage tuple
* sampling mechanism as discussed on pgsql-hackers 2004-04-02 (subject
* "Large DB"). It selects a random sample of samplesize blocks out of
* the nblocks blocks in the table. If the table has less than
* samplesize blocks, all blocks are selected.
*
* Since we know the total number of blocks in advance, we can use the
* straightforward Algorithm S from Knuth 3.4.2, rather than Vitter's
* algorithm.
*/
static void
BlockSampler_Init(BlockSampler bs, BlockNumber nblocks, int samplesize)
{
bs->N = nblocks; /* measured table size */
/*
* If we decide to reduce samplesize for tables that have less or
* not much more than samplesize blocks, here is the place to do
* it.
*/
bs->n = samplesize;
bs->t = 0; /* blocks scanned so far */
bs->m = 0; /* blocks selected so far */
}
static bool
BlockSampler_HasMore(BlockSampler bs)
{
return (bs->t < bs->N) && (bs->m < bs->n);
}
static BlockNumber
BlockSampler_Next(BlockSampler bs)
{
BlockNumber K = bs->N - bs->t; /* remaining blocks */
int k = bs->n - bs->m; /* blocks still to sample */
double p; /* probability to skip block */
double V; /* random */
Assert(BlockSampler_HasMore(bs)); /* hence K > 0 and k > 0 */
if ((BlockNumber) k >= K)
{
/* need all the rest */
bs->m++;
return bs->t++;
}
/*----------
* It is not obvious that this code matches Knuth's Algorithm S.
* Knuth says to skip the current block with probability 1 - k/K.
* If we are to skip, we should advance t (hence decrease K), and
* repeat the same probabilistic test for the next block. The naive
* implementation thus requires a random_fract() call for each block
* number. But we can reduce this to one random_fract() call per
* selected block, by noting that each time the while-test succeeds,
* we can reinterpret V as a uniform random number in the range 0 to p.
* Therefore, instead of choosing a new V, we just adjust p to be
* the appropriate fraction of its former value, and our next loop
* makes the appropriate probabilistic test.
*
* We have initially K > k > 0. If the loop reduces K to equal k,
* the next while-test must fail since p will become exactly zero
* (we assume there will not be roundoff error in the division).
* (Note: Knuth suggests a "<=" loop condition, but we use "<" just
* to be doubly sure about roundoff error.) Therefore K cannot become
* less than k, which means that we cannot fail to select enough blocks.
*----------
*/
V = random_fract();
p = 1.0 - (double) k / (double) K;
while (V < p)
{
/* skip */
bs->t++;
K--; /* keep K == N - t */
/* adjust p to be new cutoff point in reduced range */
p *= 1.0 - (double) k / (double) K;
}
/* select */
bs->m++;
return bs->t++;
}
/*
* acquire_sample_rows -- acquire a random sample of rows from the table
*
* Up to targrows rows are collected (if there are fewer than that many
* rows in the table, all rows are collected). When the table is larger
* than targrows, a truly random sample is collected: every row has an
* equal chance of ending up in the final sample.
* As of May 2004 we use a new two-stage method: Stage one selects up
* to targrows random blocks (or all blocks, if there aren't so many).
* Stage two scans these blocks and uses the Vitter algorithm to create
* a random sample of targrows rows (or less, if there are less in the
* sample of blocks). The two stages are executed simultaneously: each
* block is processed as soon as stage one returns its number and while
* the rows are read stage two controls which ones are to be inserted
* into the sample.
*
* Although every row has an equal chance of ending up in the final
* sample, this sampling method is not perfect: not every possible
* sample has an equal chance of being selected. For large relations
* the number of different blocks represented by the sample tends to be
* too small. We can live with that for now. Improvements are welcome.
*
* We also estimate the total number of rows in the table, and return that
* into *totalrows.
* into *totalrows. An important property of this sampling method is that
* because we do look at a statistically unbiased set of blocks, we should
* get an unbiased estimate of the average number of live rows per block.
* The previous sampling method put too much credence in the row density near
* the start of the table.
*
* The returned list of tuples is in order by physical position in the table.
* (We will rely on this later to derive correlation estimates.)
@ -655,101 +771,27 @@ static int
acquire_sample_rows(Relation onerel, HeapTuple *rows, int targrows,
double *totalrows)
{
int numrows = 0;
HeapScanDesc scan;
int numrows = 0; /* # rows collected */
double liverows = 0; /* # rows seen */
double deadrows = 0;
double rowstoskip = -1; /* -1 means not set yet */
BlockNumber totalblocks;
HeapTuple tuple;
ItemPointer lasttuple;
BlockNumber lastblock,
estblock;
OffsetNumber lastoffset;
int numest;
double tuplesperpage;
double t;
BlockSamplerData bs;
double rstate;
Assert(targrows > 1);
/*
* Do a simple linear scan until we reach the target number of rows.
*/
scan = heap_beginscan(onerel, SnapshotNow, 0, NULL);
totalblocks = scan->rs_nblocks; /* grab current relation size */
while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL)
{
rows[numrows++] = heap_copytuple(tuple);
if (numrows >= targrows)
break;
vacuum_delay_point();
}
heap_endscan(scan);
/*
* If we ran out of tuples then we're done, no matter how few we
* collected. No sort is needed, since they're already in order.
*/
if (!HeapTupleIsValid(tuple))
{
*totalrows = (double) numrows;
ereport(elevel,
(errmsg("\"%s\": %u pages, %d rows sampled, %.0f estimated total rows",
RelationGetRelationName(onerel),
totalblocks, numrows, *totalrows)));
return numrows;
}
/*
* Otherwise, start replacing tuples in the sample until we reach the
* end of the relation. This algorithm is from Jeff Vitter's paper
* (see full citation below). It works by repeatedly computing the
* number of the next tuple we want to fetch, which will replace a
* randomly chosen element of the reservoir (current set of tuples).
* At all times the reservoir is a true random sample of the tuples
* we've passed over so far, so when we fall off the end of the
* relation we're done.
*
* A slight difficulty is that since we don't want to fetch tuples or
* even pages that we skip over, it's not possible to fetch *exactly*
* the N'th tuple at each step --- we don't know how many valid tuples
* are on the skipped pages. We handle this by assuming that the
* average number of valid tuples/page on the pages already scanned
* over holds good for the rest of the relation as well; this lets us
* estimate which page the next tuple should be on and its position in
* the page. Then we fetch the first valid tuple at or after that
* position, being careful not to use the same tuple twice. This
* approach should still give a good random sample, although it's not
* perfect.
*/
lasttuple = &(rows[numrows - 1]->t_self);
lastblock = ItemPointerGetBlockNumber(lasttuple);
lastoffset = ItemPointerGetOffsetNumber(lasttuple);
/*
* If possible, estimate tuples/page using only completely-scanned
* pages.
*/
for (numest = numrows; numest > 0; numest--)
{
if (ItemPointerGetBlockNumber(&(rows[numest - 1]->t_self)) != lastblock)
break;
}
if (numest == 0)
{
numest = numrows; /* don't have a full page? */
estblock = lastblock + 1;
}
else
estblock = lastblock;
tuplesperpage = (double) numest / (double) estblock;
totalblocks = RelationGetNumberOfBlocks(onerel);
t = (double) numrows; /* t is the # of records processed so far */
/* Prepare for sampling block numbers */
BlockSampler_Init(&bs, totalblocks, targrows);
/* Prepare for sampling rows */
rstate = init_selection_state(targrows);
for (;;)
/* Outer loop over blocks to sample */
while (BlockSampler_HasMore(&bs))
{
double targpos;
BlockNumber targblock;
BlockNumber targblock = BlockSampler_Next(&bs);
Buffer targbuffer;
Page targpage;
OffsetNumber targoffset,
@ -757,28 +799,6 @@ acquire_sample_rows(Relation onerel, HeapTuple *rows, int targrows,
vacuum_delay_point();
t = select_next_random_record(t, targrows, &rstate);
/* Try to read the t'th record in the table */
targpos = t / tuplesperpage;
targblock = (BlockNumber) targpos;
targoffset = ((int) ((targpos - targblock) * tuplesperpage)) +
FirstOffsetNumber;
/* Make sure we are past the last selected record */
if (targblock <= lastblock)
{
targblock = lastblock;
if (targoffset <= lastoffset)
targoffset = lastoffset + 1;
}
/* Loop to find first valid record at or after given position */
pageloop:;
/*
* Have we fallen off the end of the relation?
*/
if (targblock >= totalblocks)
break;
/*
* We must maintain a pin on the target page's buffer to ensure
* that the maxoffset value stays good (else concurrent VACUUM
@ -795,62 +815,109 @@ pageloop:;
maxoffset = PageGetMaxOffsetNumber(targpage);
LockBuffer(targbuffer, BUFFER_LOCK_UNLOCK);
for (;;)
/* Inner loop over all tuples on the selected page */
for (targoffset = FirstOffsetNumber; targoffset <= maxoffset; targoffset++)
{
HeapTupleData targtuple;
Buffer tupbuffer;
if (targoffset > maxoffset)
{
/* Fell off end of this page, try next */
ReleaseBuffer(targbuffer);
targblock++;
targoffset = FirstOffsetNumber;
goto pageloop;
}
ItemPointerSet(&targtuple.t_self, targblock, targoffset);
if (heap_fetch(onerel, SnapshotNow, &targtuple, &tupbuffer,
false, NULL))
{
/*
* Found a suitable tuple, so save it, replacing one old
* tuple at random
* The first targrows live rows are simply copied into the
* reservoir.
* Then we start replacing tuples in the sample until
* we reach the end of the relation. This algorithm is
* from Jeff Vitter's paper (see full citation below).
* It works by repeatedly computing the number of tuples
* to skip before selecting a tuple, which replaces a
* randomly chosen element of the reservoir (current
* set of tuples). At all times the reservoir is a true
* random sample of the tuples we've passed over so far,
* so when we fall off the end of the relation we're done.
*/
int k = (int) (targrows * random_fract());
if (numrows < targrows)
rows[numrows++] = heap_copytuple(&targtuple);
else
{
/*
* t in Vitter's paper is the number of records already
* processed. If we need to compute a new S value, we
* must use the not-yet-incremented value of liverows
* as t.
*/
if (rowstoskip < 0)
rowstoskip = get_next_S(liverows, targrows, &rstate);
if (rowstoskip <= 0)
{
/*
* Found a suitable tuple, so save it,
* replacing one old tuple at random
*/
int k = (int) (targrows * random_fract());
Assert(k >= 0 && k < targrows);
heap_freetuple(rows[k]);
rows[k] = heap_copytuple(&targtuple);
/* this releases the second pin acquired by heap_fetch: */
Assert(k >= 0 && k < targrows);
heap_freetuple(rows[k]);
rows[k] = heap_copytuple(&targtuple);
}
rowstoskip -= 1;
}
/* must release the extra pin acquired by heap_fetch */
ReleaseBuffer(tupbuffer);
/* this releases the initial pin: */
ReleaseBuffer(targbuffer);
lastblock = targblock;
lastoffset = targoffset;
break;
liverows += 1;
}
else
{
/*
* Count dead rows, but not empty slots. This information is
* currently not used, but it seems likely we'll want it
* someday.
*/
if (targtuple.t_data != NULL)
deadrows += 1;
}
/* this tuple is dead, so advance to next one on same page */
targoffset++;
}
/* Now release the initial pin on the page */
ReleaseBuffer(targbuffer);
}
/*
* Now we need to sort the collected tuples by position (itempointer).
* If we didn't find as many tuples as we wanted then we're done.
* No sort is needed, since they're already in order.
*
* Otherwise we need to sort the collected tuples by position
* (itempointer). It's not worth worrying about corner cases
* where the tuples are already sorted.
*/
qsort((void *) rows, numrows, sizeof(HeapTuple), compare_rows);
if (numrows == targrows)
qsort((void *) rows, numrows, sizeof(HeapTuple), compare_rows);
/*
* Estimate total number of valid rows in relation.
* Estimate total number of live rows in relation.
*/
*totalrows = floor((double) totalblocks * tuplesperpage + 0.5);
if (bs.m > 0)
*totalrows = floor((liverows * totalblocks) / bs.m + 0.5);
else
*totalrows = 0.0;
/*
* Emit some interesting relation info
*/
ereport(elevel,
(errmsg("\"%s\": %u pages, %d rows sampled, %.0f estimated total rows",
(errmsg("\"%s\": scanned %d of %u pages, "
"containing %.0f live rows and %.0f dead rows; "
"%d rows in sample, %.0f estimated total rows",
RelationGetRelationName(onerel),
totalblocks, numrows, *totalrows)));
bs.m, totalblocks,
liverows, deadrows,
numrows, *totalrows)));
return numrows;
}
@ -872,23 +939,16 @@ random_fract(void)
/*
* These two routines embody Algorithm Z from "Random sampling with a
* reservoir" by Jeffrey S. Vitter, in ACM Trans. Math. Softw. 11, 1
* (Mar. 1985), Pages 37-57. While Vitter describes his algorithm in terms
* of the count S of records to skip before processing another record,
* it is convenient to work primarily with t, the index (counting from 1)
* of the last record processed and next record to process. The only extra
* state needed between calls is W, a random state variable.
*
* Note: the original algorithm defines t, S, numer, and denom as integers.
* Here we express them as doubles to avoid overflow if the number of rows
* in the table exceeds INT_MAX. The algorithm should work as long as the
* row count does not become so large that it is not represented accurately
* in a double (on IEEE-math machines this would be around 2^52 rows).
* (Mar. 1985), Pages 37-57. Vitter describes his algorithm in terms
* of the count S of records to skip before processing another record.
* It is computed primarily based on t, the number of records already read.
* The only extra state needed between calls is W, a random state variable.
*
* init_selection_state computes the initial W value.
*
* Given that we've already processed t records (t >= n),
* select_next_random_record determines the number of the next record to
* process.
* Given that we've already read t records (t >= n), get_next_S
* determines the number of records to skip before the next record is
* processed.
*/
static double
init_selection_state(int n)
@ -898,8 +958,10 @@ init_selection_state(int n)
}
static double
select_next_random_record(double t, int n, double *stateptr)
get_next_S(double t, int n, double *stateptr)
{
double S;
/* The magic constant here is T from Vitter's paper */
if (t <= (22.0 * n))
{
@ -908,11 +970,14 @@ select_next_random_record(double t, int n, double *stateptr)
quot;
V = random_fract(); /* Generate V */
S = 0;
t += 1;
/* Note: "num" in Vitter's code is always equal to t - n */
quot = (t - (double) n) / t;
/* Find min S satisfying (4.1) */
while (quot > V)
{
S += 1;
t += 1;
quot *= (t - (double) n) / t;
}
@ -922,7 +987,6 @@ select_next_random_record(double t, int n, double *stateptr)
/* Now apply Algorithm Z */
double W = *stateptr;
double term = t - (double) n + 1;
double S;
for (;;)
{
@ -970,10 +1034,9 @@ select_next_random_record(double t, int n, double *stateptr)
if (exp(log(y) / n) <= (t + X) / t)
break;
}
t += S + 1;
*stateptr = W;
}
return t;
return S;
}
/*

Write Preview
Loading…
Cancel
Save