Improved patch

[dEajL3kA]

Hello,

first of all, thank you very much for your OpenSSL 1.1 fixes in librtmp! 🎉

But I think there were some problems left. For example, you used "getter" functions to get a const pointer to internal data that really should be used "read-only", but then you assign new value to it. This causes compiler warning - for a reason 😏

I think we should use "setter" method to update internal fields of DH struct. Also fixed a few other compiler warnings.

This patch was made against the latest librtmp version (f1b83c10) taken from:
http://git.ffmpeg.org/rtmpdump

Patch:
librtmp-openssl-1.1.x-build-fixes.diff

 librtmp/dh.h        | 60 +++++++++++++++++++++++++++++++++--------------------
 librtmp/handshake.h | 14 ++++++++-----
 librtmp/hashswf.c   | 10 ++++-----
 librtmp/rtmp.c      | 10 ++++-----
 librtmp/rtmp_sys.h  |  2 +-
 rtmpsrv.c           |  6 +++---
 rtmpsuck.c          | 16 +++++++-------
 thread.c            |  6 +++---
 8 files changed, 72 insertions(+), 52 deletions(-)

diff --git a/librtmp/dh.h b/librtmp/dh.h
index 5fc3f32..28d727c 100644
--- a/librtmp/dh.h
+++ b/librtmp/dh.h
@@ -194,7 +194,7 @@ typedef BIGNUM * MP_t;
 
 /* RFC 2631, Section 2.1.5, http://www.ietf.org/rfc/rfc2631.txt */
 static int
-isValidPublicKey(MP_t y, MP_t p, MP_t q)
+isValidPublicKey(const BIGNUM *y, const BIGNUM *p, const BIGNUM *q)
 {
   int ret = TRUE;
   MP_t bn;
@@ -234,9 +234,9 @@ isValidPublicKey(MP_t y, MP_t p, MP_t q)
       MP_modexp(bn, y, q, p);
 
       if (MP_cmp_1(bn) != 0)
-	{
-	  RTMP_Log(RTMP_LOGWARNING, "DH public key does not fulfill y^q mod p = 1");
-	}
+        {
+          RTMP_Log(RTMP_LOGWARNING, "DH public key does not fulfill y^q mod p = 1");
+        }
     }
 
 failed:
@@ -253,20 +253,22 @@ DHInit(int nKeyBits)
   if (!dh)
     goto failed;
 
-  MP_new(dh->g);
+  BIGNUM *g = BN_new(); /*MP_new(dh->g);*/
 
-  if (!dh->g)
+  if (!g) /*(!dh->g)*/
     goto failed;
 
-  MP_gethex(dh->p, P1024, res);	/* prime P1024, see dhgroups.h */
+  BIGNUM *p;
+  res = BN_hex2bn(&p, P1024); /*MP_gethex(dh->p, P1024, res);*/	/* prime P1024, see dhgroups.h */
   if (!res)
     {
       goto failed;
     }
 
-  MP_set_w(dh->g, 2);	/* base 2 */
+  BN_set_word(g, 2); /*MP_set_w(dh->g, 2);*/	/* base 2 */
 
-  dh->length = nKeyBits;
+  DH_set0_pqg(dh, p, NULL, g);
+  DH_set_length(dh, nKeyBits); /*dh->length = nKeyBits;*/
   return dh;
 
 failed:
@@ -286,20 +288,24 @@ DHGenerateKey(MDH *dh)
   while (!res)
     {
       MP_t q1 = NULL;
+      const BIGNUM *p, *q, *g, *pub_key, *priv_key;
 
       if (!MDH_generate_key(dh))
-	return 0;
+        return 0;
 
       MP_gethex(q1, Q1024, res);
       assert(res);
 
-      res = isValidPublicKey(dh->pub_key, dh->p, q1);
+      DH_get0_key(dh, &pub_key, &priv_key);
+      DH_get0_pqg(dh, &p, &q, &g);
+
+      res = isValidPublicKey(pub_key, p, q1);
       if (!res)
-	{
-	  MP_free(dh->pub_key);
-	  MP_free(dh->priv_key);
-	  dh->pub_key = dh->priv_key = 0;
-	}
+        {
+          /*MP_free(dh->pub_key);*/
+          /*MP_free(dh->priv_key);*/
+          /*dh->pub_key = dh->priv_key = 0;*/
+        }
 
       MP_free(q1);
     }
@@ -311,18 +317,25 @@ DHGenerateKey(MDH *dh)
  */
 
 static int
-DHGetPublicKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen)
+DHGetPublicKey(MDH *dh, uint8_t *pubkey_out, size_t nPubkeyLen)
 {
   int len;
-  if (!dh || !dh->pub_key)
+  const BIGNUM *pub_key, *priv_key;
+
+  if (!dh) /*|| !dh->pub_key*/
+    return 0;
+
+  DH_get0_key(dh, &pub_key, &priv_key);
+  if (!pub_key)
     return 0;
 
-  len = MP_bytes(dh->pub_key);
+
+  len = MP_bytes(pub_key); /*dh->pub_key*/
   if (len <= 0 || len > (int) nPubkeyLen)
     return 0;
 
-  memset(pubkey, 0, nPubkeyLen);
-  MP_setbin(dh->pub_key, pubkey + (nPubkeyLen - len), len);
+  memset(pubkey_out, 0, nPubkeyLen);
+  BN_bn2bin(pub_key, pubkey_out + (nPubkeyLen - len)); /*MP_setbin(dh->pub_key, pubkey + (nPubkeyLen - len), len);*/
   return 1;
 }
 
@@ -353,6 +366,7 @@ DHComputeSharedSecretKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen,
   MP_t q1 = NULL, pubkeyBn = NULL;
   size_t len;
   int res;
+  const BIGNUM *p, *q, *g;
 
   if (!dh || !secret || nPubkeyLen >= INT_MAX)
     return -1;
@@ -364,7 +378,9 @@ DHComputeSharedSecretKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen,
   MP_gethex(q1, Q1024, len);
   assert(len);
 
-  if (isValidPublicKey(pubkeyBn, dh->p, q1))
+  DH_get0_pqg(dh, &p, &q, &g);
+
+  if (isValidPublicKey(pubkeyBn, p, q1))
     res = MDH_compute_key(secret, nPubkeyLen, pubkeyBn, dh);
   else
     res = -1;
diff --git a/librtmp/handshake.h b/librtmp/handshake.h
index 0438486..86d3648 100644
--- a/librtmp/handshake.h
+++ b/librtmp/handshake.h
@@ -69,9 +69,9 @@ typedef struct arcfour_ctx*	RC4_handle;
 #if OPENSSL_VERSION_NUMBER < 0x0090800 || !defined(SHA256_DIGEST_LENGTH)
 #error Your OpenSSL is too old, need 0.9.8 or newer with SHA256
 #endif
-#define HMAC_setup(ctx, key, len)	HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, key, len, EVP_sha256(), 0)
-#define HMAC_crunch(ctx, buf, len)	HMAC_Update(&ctx, buf, len)
-#define HMAC_finish(ctx, dig, dlen)	HMAC_Final(&ctx, dig, &dlen); HMAC_CTX_cleanup(&ctx)
+#define HMAC_setup(ctx, key, len)	HMAC_CTX_reset(ctx); HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0)
+#define HMAC_crunch(ctx, buf, len)	HMAC_Update(ctx, buf, len)
+#define HMAC_finish(ctx, dig, dlen)	HMAC_Final(ctx, dig, &dlen); HMAC_CTX_free(ctx)
 
 typedef RC4_KEY *	RC4_handle;
 #define RC4_alloc(h)	*h = malloc(sizeof(RC4_KEY))
@@ -117,7 +117,9 @@ static void InitRC4Encryption
 {
   uint8_t digest[SHA256_DIGEST_LENGTH];
   unsigned int digestLen = 0;
-  HMAC_CTX ctx;
+  HMAC_CTX *ctx = HMAC_CTX_new();
+  if(!ctx)
+    return;
 
   RC4_alloc(rc4keyIn);
   RC4_alloc(rc4keyOut);
@@ -266,7 +268,9 @@ HMACsha256(const uint8_t *message, size_t messageLen, const uint8_t *key,
 	   size_t keylen, uint8_t *digest)
 {
   unsigned int digestLen;
-  HMAC_CTX ctx;
+  HMAC_CTX *ctx = HMAC_CTX_new();
+  if(!ctx)
+    return;
 
   HMAC_setup(ctx, key, keylen);
   HMAC_crunch(ctx, message, messageLen);
diff --git a/librtmp/hashswf.c b/librtmp/hashswf.c
index 32b2eed..9673863 100644
--- a/librtmp/hashswf.c
+++ b/librtmp/hashswf.c
@@ -57,10 +57,10 @@
 #include <openssl/sha.h>
 #include <openssl/hmac.h>
 #include <openssl/rc4.h>
-#define HMAC_setup(ctx, key, len)	HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, (unsigned char *)key, len, EVP_sha256(), 0)
-#define HMAC_crunch(ctx, buf, len)	HMAC_Update(&ctx, (unsigned char *)buf, len)
-#define HMAC_finish(ctx, dig, dlen)	HMAC_Final(&ctx, (unsigned char *)dig, &dlen);
-#define HMAC_close(ctx)	HMAC_CTX_cleanup(&ctx)
+#define HMAC_setup(ctx, key, len)	HMAC_CTX_reset(ctx); HMAC_Init_ex(ctx, (unsigned char *)key, len, EVP_sha256(), 0)
+#define HMAC_crunch(ctx, buf, len)	HMAC_Update(ctx, (unsigned char *)buf, len)
+#define HMAC_finish(ctx, dig, dlen)	HMAC_Final(ctx, (unsigned char *)dig, &dlen);
+#define HMAC_close(ctx)	HMAC_CTX_free(ctx)
 #endif
 
 extern void RTMP_TLS_Init();
@@ -298,7 +298,7 @@ leave:
 struct info
 {
   z_stream *zs;
-  HMAC_CTX ctx;
+  HMAC_CTX *ctx;
   int first;
   int zlib;
   int size;
diff --git a/librtmp/rtmp.c b/librtmp/rtmp.c
index 0865689..df65bee 100644
--- a/librtmp/rtmp.c
+++ b/librtmp/rtmp.c
@@ -1902,7 +1902,7 @@ SendFCUnpublish(RTMP *r)
 
 SAVC(publish);
 SAVC(live);
-SAVC(record);
+/*SAVC(record);*/
 
 static int
 SendPublish(RTMP *r)
@@ -2904,8 +2904,8 @@ AVC("NetStream.Play.PublishNotify");
 static const AVal av_NetStream_Play_UnpublishNotify =
 AVC("NetStream.Play.UnpublishNotify");
 static const AVal av_NetStream_Publish_Start = AVC("NetStream.Publish.Start");
-static const AVal av_NetConnection_Connect_Rejected =
-AVC("NetConnection.Connect.Rejected");
+/*static const AVal av_NetConnection_Connect_Rejected =
+AVC("NetConnection.Connect.Rejected"); */
 
 /* Returns 0 for OK/Failed/error, 1 for 'Stop or Complete' */
 static int
@@ -3552,7 +3552,7 @@ RTMP_ReadPacket(RTMP *r, RTMPPacket *packet)
   uint8_t hbuf[RTMP_MAX_HEADER_SIZE] = { 0 };
   char *header = (char *)hbuf;
   int nSize, hSize, nToRead, nChunk;
-  int didAlloc = FALSE;
+  /*int didAlloc = FALSE;*/
   int extendedTimestamp;
 
   RTMP_Log(RTMP_LOGDEBUG2, "%s: fd=%d", __FUNCTION__, r->m_sb.sb_socket);
@@ -3679,7 +3679,7 @@ RTMP_ReadPacket(RTMP *r, RTMPPacket *packet)
 	  RTMP_Log(RTMP_LOGDEBUG, "%s, failed to allocate packet", __FUNCTION__);
 	  return FALSE;
 	}
-      didAlloc = TRUE;
+      /*didAlloc = TRUE;*/
       packet->m_headerType = (hbuf[0] & 0xc0) >> 6;
     }
 
diff --git a/librtmp/rtmp_sys.h b/librtmp/rtmp_sys.h
index 85d7e53..048f538 100644
--- a/librtmp/rtmp_sys.h
+++ b/librtmp/rtmp_sys.h
@@ -37,7 +37,7 @@
 #define GetSockError()	WSAGetLastError()
 #define SetSockError(e)	WSASetLastError(e)
 #define setsockopt(a,b,c,d,e)	(setsockopt)(a,b,c,(const char *)d,(int)e)
-#define EWOULDBLOCK	WSAETIMEDOUT	/* we don't use nonblocking, but we do use timeouts */
+/* #define EWOULDBLOCK	WSAETIMEDOUT */	/* we don't use nonblocking, but we do use timeouts */
 #define sleep(n)	Sleep(n*1000)
 #define msleep(n)	Sleep(n)
 #define SET_RCVTIMEO(tv,s)	int tv = s*1000
diff --git a/rtmpsrv.c b/rtmpsrv.c
index 5df4d3a..0a4166a 100644
--- a/rtmpsrv.c
+++ b/rtmpsrv.c
@@ -152,11 +152,11 @@ SAVC(flashVer);
 SAVC(swfUrl);
 SAVC(pageUrl);
 SAVC(tcUrl);
-SAVC(fpad);
+/*SAVC(fpad);*/
 SAVC(capabilities);
 SAVC(audioCodecs);
 SAVC(videoCodecs);
-SAVC(videoFunction);
+/*SAVC(videoFunction);*/
 SAVC(objectEncoding);
 SAVC(_result);
 SAVC(createStream);
@@ -167,7 +167,7 @@ SAVC(mode);
 SAVC(level);
 SAVC(code);
 SAVC(description);
-SAVC(secureToken);
+/*SAVC(secureToken);*/
 
 static int
 SendConnectResult(RTMP *r, double txn)
diff --git a/rtmpsuck.c b/rtmpsuck.c
index e886179..33ffff9 100644
--- a/rtmpsuck.c
+++ b/rtmpsuck.c
@@ -124,21 +124,21 @@ SAVC(flashVer);
 SAVC(swfUrl);
 SAVC(pageUrl);
 SAVC(tcUrl);
-SAVC(fpad);
-SAVC(capabilities);
+/*SAVC(fpad);*/
+/*SAVC(capabilities);*/
 SAVC(audioCodecs);
 SAVC(videoCodecs);
-SAVC(videoFunction);
+/*SAVC(videoFunction);*/
 SAVC(objectEncoding);
-SAVC(_result);
-SAVC(createStream);
+/*SAVC(_result);*/
+/*SAVC(createStream);*/
 SAVC(play);
 SAVC(closeStream);
-SAVC(fmsVer);
-SAVC(mode);
+/*SAVC(fmsVer);*/
+/*SAVC(mode);*/
 SAVC(level);
 SAVC(code);
-SAVC(secureToken);
+/*SAVC(secureToken);*/
 SAVC(onStatus);
 SAVC(close);
 static const AVal av_NetStream_Failed = AVC("NetStream.Failed");
diff --git a/thread.c b/thread.c
index 0913c98..9de42ea 100644
--- a/thread.c
+++ b/thread.c
@@ -29,13 +29,13 @@
 HANDLE
 ThreadCreate(thrfunc *routine, void *args)
 {
-  HANDLE thd;
+  uintptr_t thd;
 
-  thd = (HANDLE) _beginthread(routine, 0, args);
+  thd = _beginthread(routine, 0, args);
   if (thd == -1L)
     RTMP_LogPrintf("%s, _beginthread failed with %d\n", __FUNCTION__, errno);
 
-  return thd;
+  return (HANDLE) thd;
 }
 #else
 pthread_t

Regards.

[TNTPro]

I have used this diff with OpenSSL_1_1_0. It worked fine, thank-you.