ext/mysqlnd/mysqlnd_auth.c: Add error handling for invalid public key size (#18663)

Reported-by: Pavel Nekrasov <[email protected]>

Signed-off-by: Andrey Kovalev <[email protected]>
Co-authored-by: Andrey Kovalev <[email protected]>

Author: andr3y-k0v4l3v

ext/mysqlnd/mysqlnd_auth.c

@@ -1005,9 +1005,19 @@ void php_mysqlnd_scramble_sha2(zend_uchar * const buffer, const zend_uchar * con
 static size_t
 mysqlnd_caching_sha2_public_encrypt(MYSQLND_CONN_DATA * conn, mysqlnd_rsa_t server_public_key, size_t passwd_len, unsigned char **crypted, char *xor_str)
 {
-	size_t server_public_key_len = (size_t) EVP_PKEY_size(server_public_key);
-
 	DBG_ENTER("mysqlnd_caching_sha2_public_encrypt");
+
+	int pkey_size = EVP_PKEY_size(server_public_key);
+
+	if (pkey_size <= 0) {
+		EVP_PKEY_free(server_public_key);
+		SET_CLIENT_ERROR(conn->error_info, CR_UNKNOWN_ERROR, UNKNOWN_SQLSTATE, "invalid public key size");
+		DBG_ERR("invalid public key size");
+		DBG_RETURN(0);
+	}
+
+	size_t server_public_key_len = (size_t) pkey_size;
+
 	/*
 		Because RSA_PKCS1_OAEP_PADDING is used there is a restriction on the passwd_len.
 		RSA_PKCS1_OAEP_PADDING is recommended for new applications. See more here: