open-dsi
/
postgres
mirror of https://github.com/postgres/postgres
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Better solution to integer overflow problem in hash batch-number

Browse Source 
computation: reduce the bucket number mod nbatch.  This changes the
association between original bucket numbers and batches, but that
doesn't matter.  Minor other cleanups in hashjoin code to help
centralize decisions.
WIN32_DEV
Tom Lane 23 years ago
parent e533e7dcf5
commit a0fa0117a5
4 changed files with 60 additions and 91 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 72
      src/backend/executor/nodeHash.c
  2. 33
      src/backend/executor/nodeHashjoin.c
  3. 43
      src/backend/optimizer/path/costsize.c
  4. 3
      src/include/executor/nodeHash.h

72
src/backend/executor/nodeHash.c
Unescape View File

@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/backend/executor/nodeHash.c,v 1.72 2002/12/29 22:28:50 tgl Exp $
* $Header: /cvsroot/pgsql/src/backend/executor/nodeHash.c,v 1.73 2002/12/30 15:21:18 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@ -59,9 +59,6 @@ ExecHash(HashState *node)
outerNode = outerPlanState(node);
hashtable = node->hashtable;
if (hashtable == NULL)
elog(ERROR, "ExecHash: hash table is NULL.");
nbatch = hashtable->nbatch;
if (nbatch > 0)
@ -284,20 +281,13 @@ ExecHashTableCreate(Hash *node)
* allocate and initialize the file arrays in hashCxt
*/
hashtable->innerBatchFile = (BufFile **)
palloc(nbatch * sizeof(BufFile *));
palloc0(nbatch * sizeof(BufFile *));
hashtable->outerBatchFile = (BufFile **)
palloc(nbatch * sizeof(BufFile *));
palloc0(nbatch * sizeof(BufFile *));
hashtable->innerBatchSize = (long *)
palloc(nbatch * sizeof(long));
palloc0(nbatch * sizeof(long));
hashtable->outerBatchSize = (long *)
palloc(nbatch * sizeof(long));
for (i = 0; i < nbatch; i++)
{
hashtable->innerBatchFile[i] = NULL;
hashtable->outerBatchFile[i] = NULL;
hashtable->innerBatchSize[i] = 0;
hashtable->outerBatchSize[i] = 0;
}
palloc0(nbatch * sizeof(long));
/* The files will not be opened until later... */
}
@ -308,13 +298,7 @@ ExecHashTableCreate(Hash *node)
MemoryContextSwitchTo(hashtable->batchCxt);
hashtable->buckets = (HashJoinTuple *)
palloc(nbuckets * sizeof(HashJoinTuple));
if (hashtable->buckets == NULL)
elog(ERROR, "Insufficient memory for hash table.");
for (i = 0; i < nbuckets; i++)
hashtable->buckets[i] = NULL;
palloc0(nbuckets * sizeof(HashJoinTuple));
MemoryContextSwitchTo(oldcxt);
@ -414,15 +398,14 @@ ExecChooseHashTableSize(double ntuples, int tupwidth,
* totalbuckets/nbuckets; in fact, it is the number of groups we
* will use for the part of the data that doesn't fall into the
* first nbuckets hash buckets. We try to set it to make all the
* batches the same size. But we have to keep nbatch small
* enough to avoid integer overflow in ExecHashJoinGetBatch().
* batches the same size.
*/
dtmp = ceil((inner_rel_bytes - hash_table_bytes) /
hash_table_bytes);
if (dtmp < INT_MAX / totalbuckets)
if (dtmp < INT_MAX)
nbatch = (int) dtmp;
else
nbatch = INT_MAX / totalbuckets;
nbatch = INT_MAX;
if (nbatch <= 0)
nbatch = 1;
}
@ -481,13 +464,14 @@ ExecHashTableInsert(HashJoinTable hashtable,
List *hashkeys)
{
int bucketno = ExecHashGetBucket(hashtable, econtext, hashkeys);
int batchno = ExecHashGetBatch(bucketno, hashtable);
TupleTableSlot *slot = econtext->ecxt_innertuple;
HeapTuple heapTuple = slot->val;
/*
* decide whether to put the tuple in the hash table or a tmp file
*/
if (bucketno < hashtable->nbuckets)
if (batchno < 0)
{
/*
* put the tuple in hash table
@ -498,8 +482,6 @@ ExecHashTableInsert(HashJoinTable hashtable,
hashTupleSize = MAXALIGN(sizeof(*hashTuple)) + heapTuple->t_len;
hashTuple = (HashJoinTuple) MemoryContextAlloc(hashtable->batchCxt,
hashTupleSize);
if (hashTuple == NULL)
elog(ERROR, "Insufficient memory for hash table.");
memcpy((char *) &hashTuple->htup,
(char *) heapTuple,
sizeof(hashTuple->htup));
@ -515,11 +497,8 @@ ExecHashTableInsert(HashJoinTable hashtable,
else
{
/*
* put the tuple into a tmp file for other batches
* put the tuple into a tmp file for later batches
*/
int batchno = (hashtable->nbatch * (bucketno - hashtable->nbuckets)) /
(hashtable->totalbuckets - hashtable->nbuckets);
hashtable->innerBatchSize[batchno]++;
ExecHashJoinSaveTuple(heapTuple,
hashtable->innerBatchFile[batchno]);
@ -592,6 +571,24 @@ ExecHashGetBucket(HashJoinTable hashtable,
return bucketno;
}
/* ----------------------------------------------------------------
* ExecHashGetBatch
*
* determine the batch number for a bucketno
*
* Returns -1 if bucket belongs to initial (or current) batch,
* else 0..nbatch-1 corresponding to external batch file number for bucket.
* ----------------------------------------------------------------
*/
int
ExecHashGetBatch(int bucketno, HashJoinTable hashtable)
{
if (bucketno < hashtable->nbuckets)
return -1;
return (bucketno - hashtable->nbuckets) % hashtable->nbatch;
}
/* ----------------------------------------------------------------
* ExecScanHashBucket
*
@ -727,7 +724,6 @@ ExecHashTableReset(HashJoinTable hashtable, long ntuples)
{
MemoryContext oldcxt;
int nbuckets = hashtable->nbuckets;
int i;
/*
* Release all the hash buckets and tuples acquired in the prior pass,
@ -750,13 +746,7 @@ ExecHashTableReset(HashJoinTable hashtable, long ntuples)
/* Reallocate and reinitialize the hash bucket headers. */
hashtable->buckets = (HashJoinTuple *)
palloc(nbuckets * sizeof(HashJoinTuple));
if (hashtable->buckets == NULL)
elog(ERROR, "Insufficient memory for hash table.");
for (i = 0; i < nbuckets; i++)
hashtable->buckets[i] = NULL;
palloc0(nbuckets * sizeof(HashJoinTuple));
MemoryContextSwitchTo(oldcxt);
}

33
src/backend/executor/nodeHashjoin.c
Unescape View File

@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/backend/executor/nodeHashjoin.c,v 1.45 2002/12/15 16:17:46 tgl Exp $
* $Header: /cvsroot/pgsql/src/backend/executor/nodeHashjoin.c,v 1.46 2002/12/30 15:21:20 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@ -27,7 +27,6 @@ static TupleTableSlot *ExecHashJoinOuterGetTuple(PlanState *node,
static TupleTableSlot *ExecHashJoinGetSavedTuple(HashJoinState *hjstate,
BufFile *file,
TupleTableSlot *tupleSlot);
static int ExecHashJoinGetBatch(int bucketno, HashJoinTable hashtable);
static int ExecHashJoinNewBatch(HashJoinState *hjstate);
@ -179,17 +178,15 @@ ExecHashJoin(HashJoinState *node)
*/
if (hashtable->curbatch == 0)
{
int batch = ExecHashJoinGetBatch(node->hj_CurBucketNo,
hashtable);
int batchno = ExecHashGetBatch(node->hj_CurBucketNo,
hashtable);
if (batch > 0)
if (batchno >= 0)
{
/*
* Need to postpone this outer tuple to a later batch.
* Save it in the corresponding outer-batch file.
*/
int batchno = batch - 1;
hashtable->outerBatchSize[batchno]++;
ExecHashJoinSaveTuple(outerTupleSlot->val,
hashtable->outerBatchFile[batchno]);
@ -640,28 +637,6 @@ ExecHashJoinNewBatch(HashJoinState *hjstate)
return newbatch;
}
/* ----------------------------------------------------------------
* ExecHashJoinGetBatch
*
* determine the batch number for a bucketno
* +----------------+-------+-------+ ... +-------+
* 0 nbuckets totalbuckets
* batch 0 1 2 ...
* ----------------------------------------------------------------
*/
static int
ExecHashJoinGetBatch(int bucketno, HashJoinTable hashtable)
{
int b;
if (bucketno < hashtable->nbuckets || hashtable->nbatch == 0)
return 0;
b = (hashtable->nbatch * (bucketno - hashtable->nbuckets)) /
(hashtable->totalbuckets - hashtable->nbuckets);
return b + 1;
}
/* ----------------------------------------------------------------
* ExecHashJoinSaveTuple
*

43
src/backend/optimizer/path/costsize.c
Unescape View File

@ -42,7 +42,7 @@
* Portions Copyright (c) 1994, Regents of the University of California
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/backend/optimizer/path/costsize.c,v 1.97 2002/12/26 23:38:42 tgl Exp $
* $Header: /cvsroot/pgsql/src/backend/optimizer/path/costsize.c,v 1.98 2002/12/30 15:21:21 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@ -85,7 +85,8 @@ bool enable_mergejoin = true;
bool enable_hashjoin = true;
static Selectivity estimate_hash_bucketsize(Query *root, Var *var);
static Selectivity estimate_hash_bucketsize(Query *root, Var *var,
int nbuckets);
static bool cost_qual_eval_walker(Node *node, Cost *total);
static Selectivity approx_selectivity(Query *root, List *quals);
static void set_rel_width(Query *root, RelOptInfo *rel);
@ -882,7 +883,9 @@ cost_hashjoin(Path *path, Query *root,
outer_path->parent->width);
double innerbytes = relation_byte_size(inner_path->parent->rows,
inner_path->parent->width);
long hashtablebytes = SortMem * 1024L;
int virtualbuckets;
int physicalbuckets;
int numbatches;
Selectivity innerbucketsize;
List *hcl;
@ -898,6 +901,13 @@ cost_hashjoin(Path *path, Query *root,
startup_cost += cpu_operator_cost * inner_path->parent->rows;
run_cost += cpu_operator_cost * outer_path->parent->rows;
/* Get hash table size that executor would use for inner relation */
ExecChooseHashTableSize(inner_path->parent->rows,
inner_path->parent->width,
&virtualbuckets,
&physicalbuckets,
&numbatches);
/*
* Determine bucketsize fraction for inner relation. We use the
* smallest bucketsize estimated for any individual hashclause;
@ -931,7 +941,8 @@ cost_hashjoin(Path *path, Query *root,
if (thisbucketsize < 0)
{
/* not cached yet */
thisbucketsize = estimate_hash_bucketsize(root, right);
thisbucketsize = estimate_hash_bucketsize(root, right,
virtualbuckets);
restrictinfo->right_bucketsize = thisbucketsize;
}
}
@ -943,7 +954,8 @@ cost_hashjoin(Path *path, Query *root,
if (thisbucketsize < 0)
{
/* not cached yet */
thisbucketsize = estimate_hash_bucketsize(root, left);
thisbucketsize = estimate_hash_bucketsize(root, left,
virtualbuckets);
restrictinfo->left_bucketsize = thisbucketsize;
}
}
@ -982,7 +994,7 @@ cost_hashjoin(Path *path, Query *root,
* should be nice and sequential...). Writing the inner rel counts as
* startup cost, all the rest as run cost.
*/
if (innerbytes > hashtablebytes)
if (numbatches)
{
double outerpages = page_size(outer_path->parent->rows,
outer_path->parent->width);
@ -1019,7 +1031,7 @@ cost_hashjoin(Path *path, Query *root,
* smart enough to figure out how the restrict clauses might change the
* distribution, so this will have to do for now.
*
* We can get the number of buckets the executor will use for the given
* We are passed the number of buckets the executor will use for the given
* input relation. If the data were perfectly distributed, with the same
* number of tuples going into each available bucket, then the bucketsize
* fraction would be 1/nbuckets. But this happy state of affairs will occur
@ -1039,13 +1051,10 @@ cost_hashjoin(Path *path, Query *root,
* inner rel is well-dispersed (or the alternatives seem much worse).
*/
static Selectivity
estimate_hash_bucketsize(Query *root, Var *var)
estimate_hash_bucketsize(Query *root, Var *var, int nbuckets)
{
Oid relid;
RelOptInfo *rel;
int virtualbuckets;
int physicalbuckets;
int numbatches;
HeapTuple tuple;
Form_pg_statistic stats;
double estfract,
@ -1071,12 +1080,6 @@ estimate_hash_bucketsize(Query *root, Var *var)
if (rel->tuples <= 0.0 || rel->rows <= 0.0)
return 0.1; /* ensure we can divide below */
/* Get hash table size that executor would use for this relation */
ExecChooseHashTableSize(rel->rows, rel->width,
&virtualbuckets,
&physicalbuckets,
&numbatches);
tuple = SearchSysCache(STATRELATT,
ObjectIdGetDatum(relid),
Int16GetDatum(var->varattno),
@ -1093,7 +1096,7 @@ estimate_hash_bucketsize(Query *root, Var *var)
case ObjectIdAttributeNumber:
case SelfItemPointerAttributeNumber:
/* these are unique, so buckets should be well-distributed */
return 1.0 / (double) virtualbuckets;
return 1.0 / (double) nbuckets;
case TableOidAttributeNumber:
/* hashing this is a terrible idea... */
return 1.0;
@ -1134,8 +1137,8 @@ estimate_hash_bucketsize(Query *root, Var *var)
* the number of buckets is less than the expected number of distinct
* values; otherwise it is 1/ndistinct.
*/
if (ndistinct > (double) virtualbuckets)
estfract = 1.0 / (double) virtualbuckets;
if (ndistinct > (double) nbuckets)
estfract = 1.0 / (double) nbuckets;
else
estfract = 1.0 / ndistinct;

3
src/include/executor/nodeHash.h
Unescape View File

@ -7,7 +7,7 @@
* Portions Copyright (c) 1996-2002, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
* $Id: nodeHash.h,v 1.27 2002/12/05 15:50:37 tgl Exp $
* $Id: nodeHash.h,v 1.28 2002/12/30 15:21:23 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@ -30,6 +30,7 @@ extern void ExecHashTableInsert(HashJoinTable hashtable,
extern int ExecHashGetBucket(HashJoinTable hashtable,
ExprContext *econtext,
List *hashkeys);
extern int ExecHashGetBatch(int bucketno, HashJoinTable hashtable);
extern HeapTuple ExecScanHashBucket(HashJoinState *hjstate, List *hjclauses,
ExprContext *econtext);
extern void ExecHashTableReset(HashJoinTable hashtable, long ntuples);

Write Preview
Loading…
Cancel
Save