pkey/dh: deprecate OpenSSL::PKey::DH#generate_key!

OpenSSL 3.0.0 has made keys immutable, so PKey::DH#generate_key! can't
work on it anymore.

It's advised to use OpenSSL::PKey.generate_key instead.

Author: rhenium

lib/openssl/pkey.rb

@@ -80,14 +80,32 @@ def compute_key(pub_bn)
     # called first in order to generate the per-session keys before performing
     # the actual key exchange.
     #
+    # <b>Deprecated in version 3.0</b>. This method is incompatible with
+    # OpenSSL 3.0.0 or later.
+    #
     # See also OpenSSL::PKey.generate_key.
     #
     # Example:
-    #   dh = OpenSSL::PKey::DH.new(2048)
-    #   public_key = dh.public_key #contains no private/public key yet
-    #   public_key.generate_key!
-    #   puts public_key.private? # => true
+    #   # DEPRECATED USAGE: This will not work on OpenSSL 3.0 or later
+    #   dh0 = OpenSSL::PKey::DH.new(2048)
+    #   dh = dh0.public_key # #public_key only copies the DH parameters (contrary to the name)
+    #   dh.generate_key!
+    #   puts dh.private? # => true
+    #   puts dh0.pub_key == dh.pub_key #=> false
+    #
+    #   # With OpenSSL::PKey.generate_key
+    #   dh0 = OpenSSL::PKey::DH.new(2048)
+    #   dh = OpenSSL::PKey.generate_key(dh0)
+    #   puts dh0.pub_key == dh.pub_key #=> false
     def generate_key!
+      msg = "OpenSSL::PKey::DH is immutable on OpenSSL 3.0; " \
+        "use OpenSSL::PKey.generate_key instead"
+      if OpenSSL::OPENSSL_VERSION_NUMBER >= 0x30000000
+        raise DHError, msg
+      else
+        warn "#{caller(1, 1)[0]}: warning: #{msg}"
+      end
+
       unless priv_key
         tmp = OpenSSL::PKey.generate_key(self)
         set_key(tmp.pub_key, tmp.priv_key)

test/openssl/test_pkey_dh.rb

@@ -26,14 +26,19 @@ def test_new_break
   end
 
   def test_derive_key
-    dh1 = Fixtures.pkey("dh1024").generate_key!
-    dh2 = Fixtures.pkey("dh1024").generate_key!
+    params = Fixtures.pkey("dh1024")
+    dh1 = OpenSSL::PKey.generate_key(params)
+    dh2 = OpenSSL::PKey.generate_key(params)
     dh1_pub = OpenSSL::PKey.read(dh1.public_to_der)
     dh2_pub = OpenSSL::PKey.read(dh2.public_to_der)
+
     z = dh1.g.mod_exp(dh1.priv_key, dh1.p).mod_exp(dh2.priv_key, dh1.p).to_s(2)
     assert_equal z, dh1.derive(dh2_pub)
     assert_equal z, dh2.derive(dh1_pub)
 
+    assert_raise(OpenSSL::PKey::PKeyError) { params.derive(dh1_pub) }
+    assert_raise(OpenSSL::PKey::PKeyError) { dh1_pub.derive(params) }
+
     assert_equal z, dh1.compute_key(dh2.pub_key)
     assert_equal z, dh2.compute_key(dh1.pub_key)
   end
@@ -74,19 +79,17 @@ def test_public_key
   end
 
   def test_generate_key
-    dh = Fixtures.pkey("dh1024").public_key # creates a copy
-    assert_no_key(dh)
-    dh.generate_key!
-    assert_key(dh)
-  end
-
-  def test_key_exchange
-    dh = Fixtures.pkey("dh1024")
-    dh2 = dh.public_key
-    dh.generate_key!
-    dh2.generate_key!
-    assert_equal(dh.compute_key(dh2.pub_key), dh2.compute_key(dh.pub_key))
-  end
+    EnvUtil.suppress_warning { # Deprecated in v3.0.0; incompatible with OpenSSL 3.0
+      dh = Fixtures.pkey("dh1024").public_key # creates a copy with params only
+      assert_no_key(dh)
+      dh.generate_key!
+      assert_key(dh)
+
+      dh2 = dh.public_key
+      dh2.generate_key!
+      assert_equal(dh.compute_key(dh2.pub_key), dh2.compute_key(dh.pub_key))
+    }
+  end if !openssl?(3, 0, 0)
 
   def test_params_ok?
     dh0 = Fixtures.pkey("dh1024")