mirror of https://github.com/postgres/postgres
Tom Lane
20 years ago
parent
5d5f1a79e6
commit
80f6c35833
@ -1,22 +0,0 @@ |
||||
PROGRAM = pg_autovacuum
|
||||
OBJS = pg_autovacuum.o dllist.o
|
||||
|
||||
PG_CPPFLAGS = -I$(libpq_srcdir) -DFRONTEND
|
||||
PG_LIBS = $(libpq_pgport)
|
||||
|
||||
DOCS = README.pg_autovacuum
|
||||
|
||||
EXTRA_CLEAN = dllist.c
|
||||
|
||||
ifdef USE_PGXS |
||||
PGXS = $(shell pg_config --pgxs)
|
||||
include $(PGXS) |
||||
else |
||||
subdir = contrib/pg_autovacuum
|
||||
top_builddir = ../..
|
||||
include $(top_builddir)/src/Makefile.global |
||||
include $(top_srcdir)/contrib/contrib-global.mk |
||||
endif |
||||
|
||||
dllist.c: $(top_srcdir)/src/backend/lib/dllist.c |
||||
rm -f $@ && $(LN_S) $< .
|
||||
@ -1,246 +0,0 @@ |
||||
pg_autovacuum README |
||||
-------------------- |
||||
|
||||
pg_autovacuum is a libpq client program that monitors all the |
||||
databases associated with a PostgreSQL server. It uses the statistics |
||||
collector to monitor insert, update and delete activity. |
||||
|
||||
When a table exceeds a insert or delete threshold (for more detail on |
||||
thresholds, see "Vacuum and Analyze" below) then that table will be |
||||
vacuumed and/or analyzed. |
||||
|
||||
This allows PostgreSQL to keep the FSM (Free Space Map) and table |
||||
statistics up to date, and eliminates the need to schedule periodic |
||||
vacuums. |
||||
|
||||
The primary benefit of pg_autovacuum is that the FSM and table |
||||
statistic information are updated more nearly as frequently as needed. |
||||
When a table is actively changing, pg_autovacuum will perform the |
||||
VACUUMs and ANALYZEs that such a table needs, whereas if a table |
||||
remains static, no cycles will be wasted performing this |
||||
unnecessarily. |
||||
|
||||
A secondary benefit of pg_autovacuum is that it ensures that a |
||||
database wide vacuum is performed prior to XID wraparound. This is an |
||||
important, if rare, problem, as failing to do so can result in major |
||||
data loss. (See the section in the _Administrator's Guide_ entitled |
||||
"Preventing transaction ID wraparound failures" for more details.) |
||||
|
||||
KNOWN ISSUES: |
||||
------------- |
||||
|
||||
pg_autovacuum has been tested under Redhat Linux (by me) and Debian |
||||
GNU/Linux, Solaris, and AIX (by Christopher B. Browne) and all known |
||||
bugs have been resolved. Please report any problems to the hackers |
||||
list. |
||||
|
||||
pg_autovacuum requires that the statistics system be enabled and |
||||
reporting row level stats. The overhead of the stats system has been |
||||
shown to be significant under certain workloads. For instance, a |
||||
tight loop of queries performing "select 1" was found to run nearly |
||||
30% slower when row-level stats were enabled. However, in practice, |
||||
with more realistic workloads, the stats system overhead is usually |
||||
nominal. |
||||
|
||||
pg_autovacuum does not get started automatically by either the |
||||
postmaster or by pg_ctl. Similarly, when the postmaster exits, no one |
||||
tells pg_autovacuum. The result of that is that at the start of the |
||||
next loop, pg_autovacuum will fail to connect to the server and |
||||
exit(). Any time it fails to connect pg_autovacuum exit()s. |
||||
|
||||
While pg_autovacuum can manage vacuums for as many databases as you |
||||
may have tied to a particular PostgreSQL postmaster, it can only |
||||
connect to a single PostgreSQL postmaster. Thus, if you have multiple |
||||
postmasters on a particular host, you will need multiple pg_autovacuum |
||||
instances, and they have no way, at present, to coordinate between one |
||||
another to ensure that they do not concurrently vacuum big tables. |
||||
|
||||
When installed as a service under Windows, there is currently no way to |
||||
know the name of the PostgreSQL server service (if there even is one) |
||||
so it is not possible to specify a startup dependency. It is therefore |
||||
possible for pg_autovacuum to start before the server. |
||||
|
||||
When installed as a service under Windows, if the -P option is used to |
||||
specify the connection password, this option (and the password) is |
||||
stored in plain text in the registry. |
||||
|
||||
TODO: |
||||
----- |
||||
|
||||
At present, there are no sample scripts to automatically start up |
||||
pg_autovacuum along with the database. It would be desirable to have |
||||
a SysV script to start up pg_autovacuum after PostgreSQL has been |
||||
started. |
||||
|
||||
Some users have expressed interest in making pg_autovacuum more |
||||
configurable so that certain tables known to be inactive could be |
||||
excluded from being vacuumed. It would probably make sense to |
||||
introduce this sort of functionality by providing arguments to specify |
||||
the database and schema in which to find a configuration table. |
||||
|
||||
It would also be desirable for the daemon to monitor how busy the |
||||
system is, with a view to deferring vacuums until there is less other |
||||
activity. |
||||
|
||||
INSTALL: |
||||
-------- |
||||
|
||||
As of postgresql v7.4 pg_autovacuum is included in the main source |
||||
tree under contrib. Therefore you merely need to "make && make |
||||
install" (similar to most other contrib modules) and it will be |
||||
installed for you. |
||||
|
||||
If you are using an earlier version of PostgreSQL, uncompress the |
||||
tar.gz file into the contrib directory and modify the contrib/Makefile |
||||
to include the pg_autovacuum directory. pg_autovacuum will then be |
||||
built as part of the standard postgresql install. It is known to work |
||||
with v7.3 releases; it is not presently compatible with v7.2. |
||||
|
||||
make sure that the following are set in postgresql.conf: |
||||
|
||||
stats_start_collector = true |
||||
stats_row_level = true |
||||
|
||||
Start up the postmaster, then execute the pg_autovacuum executable. |
||||
|
||||
If you have a script that automatically starts up the PostgreSQL |
||||
instance, you might add in, after that, something similar to the |
||||
following: |
||||
|
||||
sleep 10 # To give the database some time to start up |
||||
$PGBINS/pg_autovacuum -D -s $SBASE -S $SSCALE ... [other arguments] |
||||
|
||||
Command line arguments: |
||||
----------------------- |
||||
|
||||
pg_autovacuum has the following optional arguments: |
||||
|
||||
-d debug: 0 silent, 1 basic info, 2 more debug info, etc... |
||||
-D daemonize: Detach from tty and run in background. |
||||
-s sleep base value: see "Sleeping" below. |
||||
-S sleep scaling factor: see "Sleeping" below. |
||||
-v vacuum base threshold: see "Vacuum and Analyze" below. |
||||
-V vacuum scaling factor: see "Vacuum and Analyze" below. |
||||
-a analyze base threshold: see "Vacuum and Analyze" below. |
||||
-A analyze scaling factor: see "Vacuum and Analyze" below. |
||||
-i update interval: how often (in terms of iterations of the primary loop |
||||
over the database list) to update the database list. The default is 2, |
||||
which means the list will be updated before every other pass through |
||||
the database list. |
||||
-L log file: Name of file to which output is submitted, otherwise STDERR |
||||
-U username: Username pg_autovacuum will use to connect with, if not |
||||
specified the current username is used. |
||||
-P password: Password pg_autovacuum will use to connect with. *WARNING* |
||||
This option is insecure. When installed as a Windows Service, this |
||||
option will be stored in plain text in the registry. When used with |
||||
most Unix variants, other users will be able to see the argument to |
||||
the "-P" option via ps(1). The ~/.pgpass file can be used to |
||||
specify a password more securely. |
||||
-H host: host name or IP to connect to. |
||||
-p port: port used for connection. |
||||
-h help: list of command line options. |
||||
|
||||
The following 5 autovacuum command line options correspond to the various |
||||
cost-based vacuum settings. If not given, then the cluster default values |
||||
will be used. |
||||
|
||||
-c vacuum_cost_delay |
||||
-C vacuum_cost_page_hit |
||||
-m vacuum_cost_page_miss |
||||
-n vacuum_cost_page_dirty |
||||
-l vacuum_cost_limit |
||||
|
||||
|
||||
Numerous arguments have default values defined in pg_autovacuum.h. At |
||||
the time of writing they are: |
||||
|
||||
-d 1 |
||||
-v 1000 |
||||
-V 2 |
||||
-a 500 (half of -v if not specified) |
||||
-A 1 (half of -V if not specified) |
||||
-s 300 (5 minutes) |
||||
-S 2 |
||||
-i 2 |
||||
|
||||
The following arguments are used on Windows only: |
||||
|
||||
-I Install the executable as a Windows service. Other appropriate command |
||||
line options will be stored in the registry and passed to the service |
||||
at startup. *WARNING* This includes the connection password which will |
||||
be stored in plain text. |
||||
|
||||
-N service user: Name of the Windows user account under which the service |
||||
will run. Only used when installing as a Windows service. |
||||
|
||||
-W service password: The password for the service account. Only used when |
||||
installing as a Windows service. |
||||
|
||||
-R Uninstall pg_autovacuum as a service. |
||||
|
||||
-E Dependent service that must start before this service. Normally this will be |
||||
a PostgreSQL instance, e.g. "-E pgsql-8.0.0". Only used when installing as |
||||
a Windows service. |
||||
|
||||
Vacuum and Analyze: |
||||
------------------- |
||||
|
||||
pg_autovacuum performs either a VACUUM ANALYZE or just ANALYZE |
||||
depending on the mixture of table activity (insert, update, or |
||||
delete): |
||||
|
||||
- If the number of (inserts + updates + deletes) > AnalyzeThreshold, then |
||||
only an analyze is performed. |
||||
|
||||
- If the number of (deletes + updates) > VacuumThreshold, then a |
||||
vacuum analyze is performed. |
||||
|
||||
VacuumThreshold is equal to: |
||||
vacuum_base_value + (vacuum_scaling_factor * "number of tuples in the table") |
||||
|
||||
AnalyzeThreshold is equal to: |
||||
analyze_base_value + (analyze_scaling_factor * "number of tuples in the table") |
||||
|
||||
The AnalyzeThreshold defaults to half of the VacuumThreshold since it |
||||
represents a much less expensive operation (approx 5%-10% of vacuum), |
||||
and running ANALYZE more often should not substantially degrade system |
||||
performance. |
||||
|
||||
Sleeping: |
||||
--------- |
||||
|
||||
pg_autovacuum sleeps for a while after it is done checking all the |
||||
databases. It does this in order to limit the amount of system |
||||
resources it consumes. This allows the system administrator to |
||||
configure pg_autovacuum to be more or less aggressive. |
||||
|
||||
Reducing the sleep time will cause pg_autovacuum to respond more |
||||
quickly to changes, whether they be database addition/removal, table |
||||
addition/removal, or just normal table activity. |
||||
|
||||
On the other hand, setting pg_autovacuum to sleep values too |
||||
aggressively (to too short periods of time) can have a negative effect |
||||
on server performance. For instance, if a table gets vacuumed 5 times |
||||
during the course of a large set of updates, this is likely to take a |
||||
lot more work than if the table was vacuumed just once, at the end. |
||||
|
||||
The total time it sleeps is equal to: |
||||
|
||||
base_sleep_value + sleep_scaling_factor * "duration of the previous |
||||
loop" |
||||
|
||||
Note that timing measurements are made in seconds; specifying |
||||
"pg_vacuum -s 1" means pg_autovacuum could poll the database up to 60 |
||||
times minute. In a system with large tables where vacuums may run for |
||||
several minutes, rather longer times between vacuums are likely to be |
||||
appropriate. |
||||
|
||||
What pg_autovacuum monitors: |
||||
---------------------------- |
||||
|
||||
pg_autovacuum dynamically generates a list of all databases and tables |
||||
that exist on the server. It will dynamically add and remove |
||||
databases and tables that are removed from the database server while |
||||
pg_autovacuum is running. Overhead is fairly small per object. For |
||||
example: 10 databases with 10 tables each appears to less than 10k of |
||||
memory on my Linux box. |
||||
@ -1,45 +0,0 @@ |
||||
Todo Items for pg_autovacuum client |
||||
-------------------------------------------------------------------------- |
||||
|
||||
_Add Startup Message (with datetime stamp) to Logfile when starting and logging |
||||
|
||||
_create a FSM export function and see if I can use it for pg_autovacuum |
||||
|
||||
_look into possible benifits of pgstattuple contrib work |
||||
|
||||
_Continue trying to reduce server load created by polling. |
||||
|
||||
Done: |
||||
-------------------------------------------------------------------------- |
||||
_Check if required pg_stats are enables, if not exit with error |
||||
|
||||
_Reduce the number connections and queries to the server |
||||
_Make database adding and removal part of the normal loop |
||||
_make table adding and removal part of the normal loop |
||||
|
||||
_Separate logic for vacuum and analyze |
||||
|
||||
_all pg_autovacuum specific functions are now static |
||||
|
||||
_correct usage of snprintf |
||||
|
||||
_reworked database and table update functions, now they |
||||
use the existing database connection and only one query |
||||
|
||||
_fixed -h option output |
||||
|
||||
_cleanup of 'constant == variable' used much more consistently now. |
||||
|
||||
_Guarantee database wide vacuum prior to Xid wraparound |
||||
|
||||
_change name to pg_autovacuum |
||||
|
||||
_Add proper table and database removal functions so that we can properly |
||||
clear up before we exit, and make sure we don't leak memory when removing tables and such. |
||||
|
||||
_Decouple insert and delete thresholds |
||||
|
||||
_Fix Vacuum debug routine to include the database name. |
||||
|
||||
_Allow it to detach from the tty |
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,121 +0,0 @@ |
||||
/* pg_autovacuum.h
|
||||
* Header file for pg_autovacuum.c |
||||
* (c) 2003 Matthew T. O'Connor |
||||
* |
||||
* $PostgreSQL: pgsql/contrib/pg_autovacuum/pg_autovacuum.h,v 1.15 2005/04/19 03:35:15 momjian Exp $ |
||||
*/ |
||||
|
||||
#ifndef _PG_AUTOVACUUM_H |
||||
#define _PG_AUTOVACUUM_H |
||||
|
||||
#include "libpq-fe.h" |
||||
#include "lib/dllist.h" |
||||
|
||||
#define AUTOVACUUM_DEBUG 0 |
||||
#define VACBASETHRESHOLD 1000 |
||||
#define VACSCALINGFACTOR 2 |
||||
#define SLEEPBASEVALUE 300 |
||||
#define SLEEPSCALINGFACTOR 2 |
||||
#define UPDATE_INTERVAL 2 |
||||
|
||||
/* these two constants are used to tell update_table_stats what operation we just perfomred */ |
||||
#define VACUUM_ANALYZE 0 |
||||
#define ANALYZE_ONLY 1 |
||||
|
||||
|
||||
#define TABLE_STATS_QUERY "select a.oid,a.relname,a.relnamespace,a.relpages,a.relisshared,a.reltuples,b.schemaname,b.n_tup_ins,b.n_tup_upd,b.n_tup_del from pg_class a, pg_stat_all_tables b where a.oid=b.relid and a.relkind = 'r'" |
||||
|
||||
#define PAGES_QUERY "select oid,reltuples,relpages from pg_class where oid=%u" |
||||
#define FROZENOID_QUERY "select oid,age(datfrozenxid) from pg_database where datname = 'template1'" |
||||
#define FROZENOID_QUERY2 "select oid,datname,age(datfrozenxid) from pg_database where datname!='template0'" |
||||
|
||||
/* Log levels */ |
||||
enum |
||||
{ |
||||
LVL_DEBUG = 1, |
||||
LVL_INFO, |
||||
LVL_WARNING, |
||||
LVL_ERROR, |
||||
LVL_EXTRA |
||||
}; |
||||
|
||||
/* define cmd_args stucture */ |
||||
typedef struct cmdargs |
||||
{ |
||||
int vacuum_base_threshold, |
||||
analyze_base_threshold, |
||||
update_interval, |
||||
sleep_base_value, |
||||
debug, |
||||
|
||||
/*
|
||||
* Cost-Based Vacuum Delay Settings for pg_autovacuum |
||||
*/ |
||||
av_vacuum_cost_delay, |
||||
av_vacuum_cost_page_hit, |
||||
av_vacuum_cost_page_miss, |
||||
av_vacuum_cost_page_dirty, |
||||
av_vacuum_cost_limit, |
||||
|
||||
#ifndef WIN32 |
||||
daemonize; |
||||
#else |
||||
install_as_service, |
||||
remove_as_service; |
||||
#endif |
||||
float vacuum_scaling_factor, |
||||
analyze_scaling_factor, |
||||
sleep_scaling_factor; |
||||
char *user, |
||||
*password, |
||||
#ifdef WIN32 |
||||
*service_dependencies, |
||||
*service_user, |
||||
*service_password, |
||||
#endif |
||||
*host, |
||||
*logfile, |
||||
*port; |
||||
} cmd_args; |
||||
|
||||
/*
|
||||
* Might need to add a time value for last time the whole database was |
||||
* vacuumed. We need to guarantee this happens approx every 1Billion TX's |
||||
*/ |
||||
typedef struct dbinfo |
||||
{ |
||||
Oid oid; |
||||
long age; |
||||
long analyze_threshold, |
||||
vacuum_threshold; /* Use these as defaults for table
|
||||
* thresholds */ |
||||
PGconn *conn; |
||||
char *dbname, |
||||
*username, |
||||
*password; |
||||
Dllist *table_list; |
||||
} db_info; |
||||
|
||||
typedef struct tableinfo |
||||
{ |
||||
char *schema_name, |
||||
*table_name; |
||||
float reltuples; |
||||
int relisshared; |
||||
Oid relid, |
||||
relpages; |
||||
long analyze_threshold, |
||||
vacuum_threshold; |
||||
long CountAtLastAnalyze; /* equal to: inserts + updates as
|
||||
* of the last analyze or initial |
||||
* values at startup */ |
||||
long CountAtLastVacuum; /* equal to: deletes + updates as
|
||||
* of the last vacuum or initial |
||||
* values at startup */ |
||||
long curr_analyze_count, |
||||
curr_vacuum_count; /* Latest values from stats system */ |
||||
db_info *dbi; /* pointer to the database that this table
|
||||
* belongs to */ |
||||
} tbl_info; |
||||
|
||||
#endif /* _PG_AUTOVACUUM_H */ |
||||
Loading…
Reference in new issue