open-dsi
/
postgres
mirror of https://github.com/postgres/postgres
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add VACUUM instrumentation for scanned pages, relfrozenxid.

Browse Source 
Report on scanned pages within VACUUM VERBOSE and autovacuum logging.
These are pages that were physically examined during the VACUUM
operation.  Note that this can include a small number of pages that were
marked all-visible in the visibility map by some earlier VACUUM
operation.  VACUUM won't skip all-visible pages that aren't part of a
range of all-visible pages that's at least 32 blocks in length (partly
to avoid missing out on opportunities to advance relfrozenxid during
non-aggressive VACUUMs).

Commit 44fa8488 simplified the definition of scanned pages.  It became
the complement of the pages (of those pages from rel_pages) that were
skipped using the visibility map.  And so scanned pages precisely
indicates how effective the visibility map was at saving work.  (Before
now we displayed the number of pages skipped via the visibility map when
happened to be frozen pages, but not when they were merely all-visible,
which was less useful to users.)

Rename the user-visible OldestXmin output field to "removal cutoff", and
show some supplementary information: how far behind the cutoff is
(number of XIDs behind) by the time the VACUUM operation finished.  This
will help users to figure out what's _not_ working in extreme cases
where VACUUM is fundamentally unable to remove dead tuples or freeze
older tuples (e.g., due to a leaked replication slot).  Also report when
relfrozenxid is advanced by VACUUM in output that immediately follows
"removal cutoff".  This structure is intended to highlight the
relationship between the new relfrozenxid value for the table, and the
VACUUM operation's removal cutoff.

Finally, add instrumentation of "missed dead tuples", and the number of
pages that had at least one such tuple.  These are fully DEAD (not just
RECENTLY_DEAD) tuples with storage that could not be pruned due to
failure to acquire a cleanup lock on a heap page.  This is a replacement
for the "skipped due to pin" instrumentation removed by commit 44fa8488.
It shows more details than before for pages where failing to get a
cleanup lock actually resulted in VACUUM missing out on useful work, but
usually shows nothing at all instead (the mere fact that we couldn't get
a cleanup lock is usually of no consequence whatsoever now).

Author: Peter Geoghegan <pg@bowt.ie>
Reviewed-By: Andres Freund <andres@anarazel.de>
Discussion: https://postgr.es/m/CAH2-Wznp=c=Opj8Z7RMR3G=ec3_JfGYMN_YvmCEjoPCHzWbx0g@mail.gmail.com
pull/77/head
Peter Geoghegan 3 years ago
parent 80901b3291
commit 872770fd6c
4 changed files with 82 additions and 29 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 97
      src/backend/access/heap/vacuumlazy.c
  2. 3
      src/backend/commands/analyze.c
  3. 9
      src/backend/commands/vacuum.c
  4. 2
      src/include/commands/vacuum.h

97
src/backend/access/heap/vacuumlazy.c
Unescape View File

@ -199,6 +199,7 @@ typedef struct LVRelState
BlockNumber frozenskipped_pages; /* # frozen pages skipped via VM */ BlockNumber frozenskipped_pages; /* # frozen pages skipped via VM */
BlockNumber removed_pages; /* # pages removed by relation truncation */ BlockNumber removed_pages; /* # pages removed by relation truncation */
BlockNumber lpdead_item_pages; /* # pages with LP_DEAD items */ BlockNumber lpdead_item_pages; /* # pages with LP_DEAD items */
BlockNumber missed_dead_pages; /* # pages with missed dead tuples */
BlockNumber nonempty_pages; /* actually, last nonempty page + 1 */ BlockNumber nonempty_pages; /* actually, last nonempty page + 1 */
/* Statistics output by us, for table */ /* Statistics output by us, for table */
@ -212,8 +213,8 @@ typedef struct LVRelState
/* Counters that follow are only for scanned_pages */ /* Counters that follow are only for scanned_pages */
int64 tuples_deleted; /* # deleted from table */ int64 tuples_deleted; /* # deleted from table */
int64 lpdead_items; /* # deleted from indexes */ int64 lpdead_items; /* # deleted from indexes */
int64 new_dead_tuples; /* new estimated total # of dead items in int64 recently_dead_tuples; /* # dead, but not yet removable */
* table */ int64 missed_dead_tuples; /* # removable, but not removed */
int64 num_tuples; /* total number of nonremovable tuples */ int64 num_tuples; /* total number of nonremovable tuples */
int64 live_tuples; /* live tuples (reltuples estimate) */ int64 live_tuples; /* live tuples (reltuples estimate) */
} LVRelState; } LVRelState;
@ -318,6 +319,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
write_rate; write_rate;
bool aggressive, bool aggressive,
skipwithvm; skipwithvm;
bool frozenxid_updated,
minmulti_updated;
BlockNumber orig_rel_pages; BlockNumber orig_rel_pages;
char **indnames = NULL; char **indnames = NULL;
TransactionId xidFullScanLimit; TransactionId xidFullScanLimit;
@ -539,9 +542,11 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
{ {
/* Cannot advance relfrozenxid/relminmxid */ /* Cannot advance relfrozenxid/relminmxid */
Assert(!aggressive); Assert(!aggressive);
frozenxid_updated = minmulti_updated = false;
vac_update_relstats(rel, new_rel_pages, new_live_tuples, vac_update_relstats(rel, new_rel_pages, new_live_tuples,
new_rel_allvisible, vacrel->nindexes > 0, new_rel_allvisible, vacrel->nindexes > 0,
InvalidTransactionId, InvalidMultiXactId, false); InvalidTransactionId, InvalidMultiXactId,
NULL, NULL, false);
} }
else else
{ {
@ -549,7 +554,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
orig_rel_pages); orig_rel_pages);
vac_update_relstats(rel, new_rel_pages, new_live_tuples, vac_update_relstats(rel, new_rel_pages, new_live_tuples,
new_rel_allvisible, vacrel->nindexes > 0, new_rel_allvisible, vacrel->nindexes > 0,
FreezeLimit, MultiXactCutoff, false); FreezeLimit, MultiXactCutoff,
&frozenxid_updated, &minmulti_updated, false);
} }
/* /*
@ -565,7 +571,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
pgstat_report_vacuum(RelationGetRelid(rel), pgstat_report_vacuum(RelationGetRelid(rel),
rel->rd_rel->relisshared, rel->rd_rel->relisshared,
Max(new_live_tuples, 0), Max(new_live_tuples, 0),
vacrel->new_dead_tuples); vacrel->recently_dead_tuples +
vacrel->missed_dead_tuples);
pgstat_progress_end_command(); pgstat_progress_end_command();
if (instrument) if (instrument)
@ -578,6 +585,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
{ {
StringInfoData buf; StringInfoData buf;
char *msgfmt; char *msgfmt;
int32 diff;
TimestampDifference(starttime, endtime, &secs, &usecs); TimestampDifference(starttime, endtime, &secs, &usecs);
@ -629,16 +637,40 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
vacrel->relnamespace, vacrel->relnamespace,
vacrel->relname, vacrel->relname,
vacrel->num_index_scans); vacrel->num_index_scans);
appendStringInfo(&buf, _("pages: %u removed, %u remain, %u skipped frozen\n"), appendStringInfo(&buf, _("pages: %u removed, %u remain, %u scanned (%.2f%% of total)\n"),
vacrel->removed_pages, vacrel->removed_pages,
vacrel->rel_pages, vacrel->rel_pages,
vacrel->frozenskipped_pages); vacrel->scanned_pages,
orig_rel_pages == 0 ? 0 :
100.0 * vacrel->scanned_pages / orig_rel_pages);
appendStringInfo(&buf, appendStringInfo(&buf,
_("tuples: %lld removed, %lld remain, %lld are dead but not yet removable, oldest xmin: %u\n"), _("tuples: %lld removed, %lld remain, %lld are dead but not yet removable\n"),
(long long) vacrel->tuples_deleted, (long long) vacrel->tuples_deleted,
(long long) vacrel->new_rel_tuples, (long long) vacrel->new_rel_tuples,
(long long) vacrel->new_dead_tuples, (long long) vacrel->recently_dead_tuples);
OldestXmin); if (vacrel->missed_dead_tuples > 0)
appendStringInfo(&buf,
_("tuples missed: %lld dead from %u pages not removed due to cleanup lock contention\n"),
(long long) vacrel->missed_dead_tuples,
vacrel->missed_dead_pages);
diff = (int32) (ReadNextTransactionId() - OldestXmin);
appendStringInfo(&buf,
_("removable cutoff: %u, older by %d xids when operation ended\n"),
OldestXmin, diff);
if (frozenxid_updated)
{
diff = (int32) (FreezeLimit - vacrel->relfrozenxid);
appendStringInfo(&buf,
_("new relfrozenxid: %u, which is %d xids ahead of previous value\n"),
FreezeLimit, diff);
}
if (minmulti_updated)
{
diff = (int32) (MultiXactCutoff - vacrel->relminmxid);
appendStringInfo(&buf,
_("new relminmxid: %u, which is %d mxids ahead of previous value\n"),
MultiXactCutoff, diff);
}
if (orig_rel_pages > 0) if (orig_rel_pages > 0)
{ {
if (vacrel->do_index_vacuuming) if (vacrel->do_index_vacuuming)
@ -779,13 +811,15 @@ lazy_scan_heap(LVRelState *vacrel, int nworkers)
vacrel->frozenskipped_pages = 0; vacrel->frozenskipped_pages = 0;
vacrel->removed_pages = 0; vacrel->removed_pages = 0;
vacrel->lpdead_item_pages = 0; vacrel->lpdead_item_pages = 0;
vacrel->missed_dead_pages = 0;
vacrel->nonempty_pages = 0; vacrel->nonempty_pages = 0;
/* Initialize instrumentation counters */ /* Initialize instrumentation counters */
vacrel->num_index_scans = 0; vacrel->num_index_scans = 0;
vacrel->tuples_deleted = 0; vacrel->tuples_deleted = 0;
vacrel->lpdead_items = 0; vacrel->lpdead_items = 0;
vacrel->new_dead_tuples = 0; vacrel->recently_dead_tuples = 0;
vacrel->missed_dead_tuples = 0;
vacrel->num_tuples = 0; vacrel->num_tuples = 0;
vacrel->live_tuples = 0; vacrel->live_tuples = 0;
@ -1331,7 +1365,8 @@ lazy_scan_heap(LVRelState *vacrel, int nworkers)
* (unlikely) scenario that new_live_tuples is -1, take it as zero. * (unlikely) scenario that new_live_tuples is -1, take it as zero.
*/ */
vacrel->new_rel_tuples = vacrel->new_rel_tuples =
Max(vacrel->new_live_tuples, 0) + vacrel->new_dead_tuples; Max(vacrel->new_live_tuples, 0) + vacrel->recently_dead_tuples +
vacrel->missed_dead_tuples;
/* /*
* Release any remaining pin on visibility map page. * Release any remaining pin on visibility map page.
@ -1539,7 +1574,7 @@ lazy_scan_prune(LVRelState *vacrel,
HTSV_Result res; HTSV_Result res;
int tuples_deleted, int tuples_deleted,
lpdead_items, lpdead_items,
new_dead_tuples, recently_dead_tuples,
num_tuples, num_tuples,
live_tuples; live_tuples;
int nnewlpdead; int nnewlpdead;
@ -1556,7 +1591,7 @@ retry:
/* Initialize (or reset) page-level counters */ /* Initialize (or reset) page-level counters */
tuples_deleted = 0; tuples_deleted = 0;
lpdead_items = 0; lpdead_items = 0;
new_dead_tuples = 0; recently_dead_tuples = 0;
num_tuples = 0; num_tuples = 0;
live_tuples = 0; live_tuples = 0;
@ -1715,11 +1750,11 @@ retry:
case HEAPTUPLE_RECENTLY_DEAD: case HEAPTUPLE_RECENTLY_DEAD:
/* /*
* If tuple is recently deleted then we must not remove it * If tuple is recently dead then we must not remove it from
* from relation. (We only remove items that are LP_DEAD from * the relation. (We only remove items that are LP_DEAD from
* pruning.) * pruning.)
*/ */
new_dead_tuples++; recently_dead_tuples++;
prunestate->all_visible = false; prunestate->all_visible = false;
break; break;
case HEAPTUPLE_INSERT_IN_PROGRESS: case HEAPTUPLE_INSERT_IN_PROGRESS:
@ -1895,7 +1930,7 @@ retry:
/* Finally, add page-local counts to whole-VACUUM counts */ /* Finally, add page-local counts to whole-VACUUM counts */
vacrel->tuples_deleted += tuples_deleted; vacrel->tuples_deleted += tuples_deleted;
vacrel->lpdead_items += lpdead_items; vacrel->lpdead_items += lpdead_items;
vacrel->new_dead_tuples += new_dead_tuples; vacrel->recently_dead_tuples += recently_dead_tuples;
vacrel->num_tuples += num_tuples; vacrel->num_tuples += num_tuples;
vacrel->live_tuples += live_tuples; vacrel->live_tuples += live_tuples;
} }
@ -1932,7 +1967,8 @@ lazy_scan_noprune(LVRelState *vacrel,
int lpdead_items, int lpdead_items,
num_tuples, num_tuples,
live_tuples, live_tuples,
new_dead_tuples; recently_dead_tuples,
missed_dead_tuples;
HeapTupleHeader tupleheader; HeapTupleHeader tupleheader;
OffsetNumber deadoffsets[MaxHeapTuplesPerPage]; OffsetNumber deadoffsets[MaxHeapTuplesPerPage];
@ -1944,7 +1980,8 @@ lazy_scan_noprune(LVRelState *vacrel,
lpdead_items = 0; lpdead_items = 0;
num_tuples = 0; num_tuples = 0;
live_tuples = 0; live_tuples = 0;
new_dead_tuples = 0; recently_dead_tuples = 0;
missed_dead_tuples = 0;
maxoff = PageGetMaxOffsetNumber(page); maxoff = PageGetMaxOffsetNumber(page);
for (offnum = FirstOffsetNumber; for (offnum = FirstOffsetNumber;
@ -2018,16 +2055,15 @@ lazy_scan_noprune(LVRelState *vacrel,
/* /*
* There is some useful work for pruning to do, that won't be * There is some useful work for pruning to do, that won't be
* done due to failure to get a cleanup lock. * done due to failure to get a cleanup lock.
*
* TODO Add dedicated instrumentation for this case
*/ */
missed_dead_tuples++;
break; break;
case HEAPTUPLE_RECENTLY_DEAD: case HEAPTUPLE_RECENTLY_DEAD:
/* /*
* Count in new_dead_tuples, just like lazy_scan_prune * Count in recently_dead_tuples, just like lazy_scan_prune
*/ */
new_dead_tuples++; recently_dead_tuples++;
break; break;
case HEAPTUPLE_INSERT_IN_PROGRESS: case HEAPTUPLE_INSERT_IN_PROGRESS:
@ -2063,7 +2099,7 @@ lazy_scan_noprune(LVRelState *vacrel,
*/ */
*hastup = true; *hastup = true;
num_tuples += lpdead_items; num_tuples += lpdead_items;
/* TODO HEAPTUPLE_DEAD style instrumentation needed here, too */ missed_dead_tuples += lpdead_items;
} }
*recordfreespace = true; *recordfreespace = true;
@ -2112,9 +2148,12 @@ lazy_scan_noprune(LVRelState *vacrel,
/* /*
* Finally, add relevant page-local counts to whole-VACUUM counts * Finally, add relevant page-local counts to whole-VACUUM counts
*/ */
vacrel->new_dead_tuples += new_dead_tuples; vacrel->recently_dead_tuples += recently_dead_tuples;
vacrel->missed_dead_tuples += missed_dead_tuples;
vacrel->num_tuples += num_tuples; vacrel->num_tuples += num_tuples;
vacrel->live_tuples += live_tuples; vacrel->live_tuples += live_tuples;
if (missed_dead_tuples > 0)
vacrel->missed_dead_pages++;
/* Caller won't need to call lazy_scan_prune with same page */ /* Caller won't need to call lazy_scan_prune with same page */
return true; return true;
@ -2193,8 +2232,8 @@ lazy_vacuum(LVRelState *vacrel)
* dead_items space is not CPU cache resident. * dead_items space is not CPU cache resident.
* *
* We don't take any special steps to remember the LP_DEAD items (such * We don't take any special steps to remember the LP_DEAD items (such
* as counting them in new_dead_tuples report to the stats collector) * as counting them in our final report to the stats collector) when
* when the optimization is applied. Though the accounting used in * the optimization is applied. Though the accounting used in
* analyze.c's acquire_sample_rows() will recognize the same LP_DEAD * analyze.c's acquire_sample_rows() will recognize the same LP_DEAD
* items as dead rows in its own stats collector report, that's okay. * items as dead rows in its own stats collector report, that's okay.
* The discrepancy should be negligible. If this optimization is ever * The discrepancy should be negligible. If this optimization is ever
@ -3321,7 +3360,7 @@ update_index_statistics(LVRelState *vacrel)
false, false,
InvalidTransactionId, InvalidTransactionId,
InvalidMultiXactId, InvalidMultiXactId,
false); NULL, NULL, false);
} }
} }

3
src/backend/commands/analyze.c
Unescape View File

@ -645,6 +645,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params,
hasindex, hasindex,
InvalidTransactionId, InvalidTransactionId,
InvalidMultiXactId, InvalidMultiXactId,
NULL, NULL,
in_outer_xact); in_outer_xact);
/* Same for indexes */ /* Same for indexes */
@ -661,6 +662,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params,
false, false,
InvalidTransactionId, InvalidTransactionId,
InvalidMultiXactId, InvalidMultiXactId,
NULL, NULL,
in_outer_xact); in_outer_xact);
} }
} }
@ -673,6 +675,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params,
vac_update_relstats(onerel, -1, totalrows, vac_update_relstats(onerel, -1, totalrows,
0, hasindex, InvalidTransactionId, 0, hasindex, InvalidTransactionId,
InvalidMultiXactId, InvalidMultiXactId,
NULL, NULL,
in_outer_xact); in_outer_xact);
} }

9
src/backend/commands/vacuum.c
Unescape View File

@ -1315,6 +1315,7 @@ vac_update_relstats(Relation relation,
BlockNumber num_all_visible_pages, BlockNumber num_all_visible_pages,
bool hasindex, TransactionId frozenxid, bool hasindex, TransactionId frozenxid,
MultiXactId minmulti, MultiXactId minmulti,
bool *frozenxid_updated, bool *minmulti_updated,
bool in_outer_xact) bool in_outer_xact)
{ {
Oid relid = RelationGetRelid(relation); Oid relid = RelationGetRelid(relation);
@ -1390,22 +1391,30 @@ vac_update_relstats(Relation relation,
* This should match vac_update_datfrozenxid() concerning what we consider * This should match vac_update_datfrozenxid() concerning what we consider
* to be "in the future". * to be "in the future".
*/ */
if (frozenxid_updated)
*frozenxid_updated = false;
if (TransactionIdIsNormal(frozenxid) && if (TransactionIdIsNormal(frozenxid) &&
pgcform->relfrozenxid != frozenxid && pgcform->relfrozenxid != frozenxid &&
(TransactionIdPrecedes(pgcform->relfrozenxid, frozenxid) || (TransactionIdPrecedes(pgcform->relfrozenxid, frozenxid) ||
TransactionIdPrecedes(ReadNextTransactionId(), TransactionIdPrecedes(ReadNextTransactionId(),
pgcform->relfrozenxid))) pgcform->relfrozenxid)))
{ {
if (frozenxid_updated)
*frozenxid_updated = true;
pgcform->relfrozenxid = frozenxid; pgcform->relfrozenxid = frozenxid;
dirty = true; dirty = true;
} }
/* Similarly for relminmxid */ /* Similarly for relminmxid */
if (minmulti_updated)
*minmulti_updated = false;
if (MultiXactIdIsValid(minmulti) && if (MultiXactIdIsValid(minmulti) &&
pgcform->relminmxid != minmulti && pgcform->relminmxid != minmulti &&
(MultiXactIdPrecedes(pgcform->relminmxid, minmulti) || (MultiXactIdPrecedes(pgcform->relminmxid, minmulti) ||
MultiXactIdPrecedes(ReadNextMultiXactId(), pgcform->relminmxid))) MultiXactIdPrecedes(ReadNextMultiXactId(), pgcform->relminmxid)))
{ {
if (minmulti_updated)
*minmulti_updated = true;
pgcform->relminmxid = minmulti; pgcform->relminmxid = minmulti;
dirty = true; dirty = true;
} }

2
src/include/commands/vacuum.h
Unescape View File

@ -283,6 +283,8 @@ extern void vac_update_relstats(Relation relation,
bool hasindex, bool hasindex,
TransactionId frozenxid, TransactionId frozenxid,
MultiXactId minmulti, MultiXactId minmulti,
bool *frozenxid_updated,
bool *minmulti_updated,
bool in_outer_xact); bool in_outer_xact);
extern void vacuum_set_xid_limits(Relation rel, extern void vacuum_set_xid_limits(Relation rel,
int freeze_min_age, int freeze_table_age, int freeze_min_age, int freeze_table_age,

Write Preview
Loading…
Cancel
Save