PKGBUILDs/alarm/uboot-tools/0001-rsa-Fix-build-with-OpenSSL-1.1.x.patch

From d3afa5f4808b051411b286029aeeb449d460ecc7 Mon Sep 17 00:00:00 2001
From: Jelle van der Waa <jelle@vdwaa.nl>
Date: Tue, 4 Apr 2017 23:59:48 +0200
Subject: [PATCH 1/3] rsa: Fix build with OpenSSL 1.1.x

The rsa_st struct has been made opaque in 1.1.x, add forward compatible
code to access the n, e, d members of rsa_struct.

EVP_MD_CTX_cleanup has been removed in 1.1.x and EVP_MD_CTX_reset should be
called to reinitialise an already created structure.

Tested-by: Peter Robinson <pbrobinson@gmail.com>
---
 lib/rsa/rsa-sign.c | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/lib/rsa/rsa-sign.c b/lib/rsa/rsa-sign.c
index 8c6637e328..5b7022d428 100644
--- a/lib/rsa/rsa-sign.c
+++ b/lib/rsa/rsa-sign.c
@@ -15,11 +15,25 @@
 #include <openssl/ssl.h>
 #include <openssl/evp.h>
 #include <openssl/engine.h>
+#include <openssl/bn.h>
 
 #if OPENSSL_VERSION_NUMBER >= 0x10000000L
 #define HAVE_ERR_REMOVE_THREAD_STATE
 #endif
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+static void RSA_get0_key(const RSA *r,
+                 const BIGNUM **n, const BIGNUM **e, const BIGNUM **d)
+{
+   if (n != NULL)
+       *n = r->n;
+   if (e != NULL)
+       *e = r->e;
+   if (d != NULL)
+       *d = r->d;
+}
+#endif
+
 static int rsa_err(const char *msg)
 {
 	unsigned long sslErr = ERR_get_error();
@@ -286,7 +300,7 @@ static int rsa_init(void)
 {
 	int ret;
 
-	ret = SSL_library_init();
+	ret = OPENSSL_init_ssl(0, NULL);
 	if (!ret) {
 		fprintf(stderr, "Failure to init SSL library\n");
 		return -1;
@@ -409,7 +423,11 @@ static int rsa_sign_with_key(RSA *rsa, struct checksum_algo *checksum_algo,
 		ret = rsa_err("Could not obtain signature");
 		goto err_sign;
 	}
-	EVP_MD_CTX_cleanup(context);
+	#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+		EVP_MD_CTX_cleanup(context);
+	#else
+		EVP_MD_CTX_reset(context);
+	#endif
 	EVP_MD_CTX_destroy(context);
 	EVP_PKEY_free(key);
 
@@ -479,6 +497,7 @@ static int rsa_get_exponent(RSA *key, uint64_t *e)
 {
 	int ret;
 	BIGNUM *bn_te;
+	const BIGNUM *key_e;
 	uint64_t te;
 
 	ret = -EINVAL;
@@ -487,17 +506,18 @@ static int rsa_get_exponent(RSA *key, uint64_t *e)
 	if (!e)
 		goto cleanup;
 
-	if (BN_num_bits(key->e) > 64)
+	RSA_get0_key(key, NULL, &key_e, NULL);
+	if (BN_num_bits(key_e) > 64)
 		goto cleanup;
 
-	*e = BN_get_word(key->e);
+	*e = BN_get_word(key_e);
 
-	if (BN_num_bits(key->e) < 33) {
+	if (BN_num_bits(key_e) < 33) {
 		ret = 0;
 		goto cleanup;
 	}
 
-	bn_te = BN_dup(key->e);
+	bn_te = BN_dup(key_e);
 	if (!bn_te)
 		goto cleanup;
 
@@ -527,6 +547,7 @@ int rsa_get_params(RSA *key, uint64_t *exponent, uint32_t *n0_invp,
 {
 	BIGNUM *big1, *big2, *big32, *big2_32;
 	BIGNUM *n, *r, *r_squared, *tmp;
+	const BIGNUM *key_n;
 	BN_CTX *bn_ctx = BN_CTX_new();
 	int ret = 0;
 
@@ -548,7 +569,8 @@ int rsa_get_params(RSA *key, uint64_t *exponent, uint32_t *n0_invp,
 	if (0 != rsa_get_exponent(key, exponent))
 		ret = -1;
 
-	if (!BN_copy(n, key->n) || !BN_set_word(big1, 1L) ||
+	RSA_get0_key(key, &key_n, NULL, NULL);
+	if (!BN_copy(n, key_n) || !BN_set_word(big1, 1L) ||
 	    !BN_set_word(big2, 2L) || !BN_set_word(big32, 32L))
 		ret = -1;
 
-- 
2.12.2