Fix assorted integer-overflow hazards in varbit.c.

bitshiftright() and bitshiftleft() would recursively call each other
infinitely if the user passed INT_MIN for the shift amount, due to integer
overflow in negating the shift amount.  To fix, clamp to -VARBITMAXLEN.
That doesn't change the results since any shift distance larger than the
input bit string's length produces an all-zeroes result.

Also fix some places that seemed inadequately paranoid about input typmods
exceeding VARBITMAXLEN.  While a typmod accepted by anybit_typmodin() will
certainly be much less than that, at least some of these spots are
reachable with user-chosen integer values.

Andreas Seltenreich and Tom Lane

Discussion: <87d1j2zqtz.fsf@credativ.de>

src/backend/utils/adt/varbit.c

@@ -304,7 +304,7 @@ bit_recv(PG_FUNCTION_ARGS)
 	bits8		mask;
 
 	bitlen = pq_getmsgint(buf, sizeof(int32));
-	if (bitlen < 0)
+	if (bitlen < 0 || bitlen > VARBITMAXLEN)
 		ereport(ERROR,
 				(errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
 				 errmsg("invalid length in external bit string")));
@@ -367,7 +367,7 @@ bit(PG_FUNCTION_ARGS)
 	bits8		mask;
 
 	/* No work if typmod is invalid or supplied data matches it already */
-	if (len <= 0 || len == VARBITLEN(arg))
+	if (len <= 0 || len > VARBITMAXLEN || len == VARBITLEN(arg))
 		PG_RETURN_VARBIT_P(arg);
 
 	if (!isExplicit)
@@ -620,7 +620,7 @@ varbit_recv(PG_FUNCTION_ARGS)
 	bits8		mask;
 
 	bitlen = pq_getmsgint(buf, sizeof(int32));
-	if (bitlen < 0)
+	if (bitlen < 0 || bitlen > VARBITMAXLEN)
 		ereport(ERROR,
 				(errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
 				 errmsg("invalid length in external bit string")));
@@ -1353,9 +1353,14 @@ bitshiftleft(PG_FUNCTION_ARGS)
 
 	/* Negative shift is a shift to the right */
 	if (shft < 0)
+	{
+		/* Prevent integer overflow in negation */
+		if (shft < -VARBITMAXLEN)
+			shft = -VARBITMAXLEN;
 		PG_RETURN_DATUM(DirectFunctionCall2(bitshiftright,
 											VarBitPGetDatum(arg),
 											Int32GetDatum(-shft)));
+	}
 
 	result = (VarBit *) palloc(VARSIZE(arg));
 	SET_VARSIZE(result, VARSIZE(arg));
@@ -1413,9 +1418,14 @@ bitshiftright(PG_FUNCTION_ARGS)
 
 	/* Negative shift is a shift to the left */
 	if (shft < 0)
+	{
+		/* Prevent integer overflow in negation */
+		if (shft < -VARBITMAXLEN)
+			shft = -VARBITMAXLEN;
 		PG_RETURN_DATUM(DirectFunctionCall2(bitshiftleft,
 											VarBitPGetDatum(arg),
 											Int32GetDatum(-shft)));
+	}
 
 	result = (VarBit *) palloc(VARSIZE(arg));
 	SET_VARSIZE(result, VARSIZE(arg));
@@ -1473,7 +1483,7 @@ bitfromint4(PG_FUNCTION_ARGS)
 	int			destbitsleft,
 				srcbitsleft;
 
-	if (typmod <= 0)
+	if (typmod <= 0 || typmod > VARBITMAXLEN)
 		typmod = 1;				/* default bit length */
 
 	rlen = VARBITTOTALLEN(typmod);
@@ -1553,7 +1563,7 @@ bitfromint8(PG_FUNCTION_ARGS)
 	int			destbitsleft,
 				srcbitsleft;
 
-	if (typmod <= 0)
+	if (typmod <= 0 || typmod > VARBITMAXLEN)
 		typmod = 1;				/* default bit length */
 
 	rlen = VARBITTOTALLEN(typmod);