Fix possible buffer overrun in contrib/pg_trgm.

Allow for the possibility that folding a string to lower case makes it
longer (due to replacing a character with a longer multibyte character).
This doesn't change the number of trigrams that will be extracted, but
it does affect the required size of an intermediate buffer in
generate_trgm().  Per bug #8821 from Ufuk Kayserilioglu.

Also install some checks that the input string length is not so large
as to cause overflow in the calculations of palloc request sizes.

Back-patch to all supported versions.

contrib/pg_trgm/trgm_op.c

@@ -3,9 +3,10 @@
  */
 #include "trgm.h"
 #include <ctype.h>
-#include "utils/array.h"
 #include "catalog/pg_type.h"
 #include "tsearch/ts_locale.h"
+#include "utils/array.h"
+#include "utils/memutils.h"
 
 PG_MODULE_MAGIC;
 
@@ -172,6 +173,18 @@ generate_trgm(char *str, int slen)
 	char	   *bword,
 			   *eword;
 
+	/*
+	 * Guard against possible overflow in the palloc requests below.  (We
+	 * don't worry about the additive constants, since palloc can detect
+	 * requests that are a little above MaxAllocSize --- we just need to
+	 * prevent integer overflow in the multiplications.)
+	 */
+	if ((Size) (slen / 2) >= (MaxAllocSize / (sizeof(trgm) * 3)) ||
+		(Size) slen >= (MaxAllocSize / pg_database_encoding_max_length()))
+		ereport(ERROR,
+				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+				 errmsg("out of memory")));
+
 	trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) *3);
 	trg->flag = ARRKEY;
 	SET_VARSIZE(trg, TRGMHDRSIZE);
@@ -181,7 +194,8 @@ generate_trgm(char *str, int slen)
 
 	tptr = GETARR(trg);
 
-	buf = palloc(sizeof(char) * (slen + 4));
+	/* Allocate a buffer for case-folded, blank-padded words */
+	buf = (char *) palloc(slen * pg_database_encoding_max_length() + 4);
 
 	if (LPADDING > 0)
 	{
@@ -205,6 +219,7 @@ generate_trgm(char *str, int slen)
 #ifdef IGNORECASE
 		pfree(bword);
 #endif
+
 		buf[LPADDING + bytelen] = ' ';
 		buf[LPADDING + bytelen + 1] = ' ';
 
@@ -220,7 +235,10 @@ generate_trgm(char *str, int slen)
 	if ((len = tptr - GETARR(trg)) == 0)
 		return trg;
 
-	if (len > 0)
+	/*
+	 * Make trigrams unique.
+	 */
+	if (len > 1)
 	{
 		qsort((void *) GETARR(trg), len, sizeof(trgm), comp_trgm);
 		len = unique_array(GETARR(trg), len);