Fix errormessage for missing system CA in OpenSSL 3.1

The error message for a missing or invalid system CA when using
sslrootcert=system differs based on the OpenSSL version used.

In OpenSSL 1.0.1-3.0 it is reported as SSL Error, with varying
degrees of helpfulness in the error message. With OpenSSL 3.1 it
is reported as an SSL SYSCALL error with "Undefined error" as
the error message. This fix pulls out the particular error in
OpenSSL 3.1 as a certificate verify error in order to help the
user better figure out what happened, and to keep the ssl test
working. While there is no evidence that extracing the errors
will clobber errno, this adds a guard against that regardless
to also make the consistent with how we handle OpenSSL errors
elsewhere. It also memorizes the output from OpenSSL 3.0 in
the test in cases where the system CA isn't responding.

Reported-by: Peter Eisentraut <peter.eisentraut@enterprisedb.com>
Discussion: https://postgr.es/m/c39be3c5-c1a5-1e33-1024-16f527e251a4@enterprisedb.com

src/interfaces/libpq/fe-secure-openssl.c

@@ -1489,10 +1489,12 @@ open_client_SSL(PGconn *conn)
 {
 	int			r;
 
+	SOCK_ERRNO_SET(0);
 	ERR_clear_error();
 	r = SSL_connect(conn->ssl);
 	if (r <= 0)
 	{
+		int			save_errno = SOCK_ERRNO;
 		int			err = SSL_get_error(conn->ssl, r);
 		unsigned long ecode;
 
@@ -1508,10 +1510,26 @@ open_client_SSL(PGconn *conn)
 			case SSL_ERROR_SYSCALL:
 				{
 					char		sebuf[PG_STRERROR_R_BUFLEN];
-
-					if (r == -1)
+					unsigned long vcode;
+
+					vcode = SSL_get_verify_result(conn->ssl);
+
+					/*
+					 * If we get an X509 error here for failing to load the
+					 * local issuer cert, without an error in the socket layer
+					 * it means that verification failed due to a missing
+					 * system CA pool without it being a protocol error. We
+					 * inspect the sslrootcert setting to ensure that the user
+					 * was using the system CA pool. For other errors, log them
+					 * using the normal SYSCALL logging.
+					 */
+					if (!save_errno && vcode == X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY &&
+						strcmp(conn->sslrootcert, "system") == 0)
+						libpq_append_conn_error(conn, "SSL error: certificate verify failed: %s",
+												X509_verify_cert_error_string(vcode));
+					else if (r == -1)
 						libpq_append_conn_error(conn, "SSL SYSCALL error: %s",
-										  SOCK_STRERROR(SOCK_ERRNO, sebuf, sizeof(sebuf)));
+										  SOCK_STRERROR(save_errno, sebuf, sizeof(sebuf)));
 					else
 						libpq_append_conn_error(conn, "SSL SYSCALL error: EOF detected");
 					pgtls_close(conn);

src/test/ssl/t/001_ssltests.pl

@@ -476,10 +476,12 @@ $common_connstr =
   "$default_ssl_connstr user=ssltestuser dbname=trustdb sslrootcert=system hostaddr=$SERVERHOSTADDR";
 
 # By default our custom-CA-signed certificate should not be trusted.
+# OpenSSL 3.0 reports a missing/invalid system CA as "unregistered schema"
+# instead of a failed certificate verification.
 $node->connect_fails(
 	"$common_connstr sslmode=verify-full host=common-name.pg-ssltest.test",
 	"sslrootcert=system does not connect with private CA",
-	expected_stderr => qr/SSL error: certificate verify failed/);
+	expected_stderr => qr/SSL error: (certificate verify failed|unregistered scheme)/);
 
 # Modes other than verify-full cannot be mixed with sslrootcert=system.
 $node->connect_fails(