diff --git a/build.gradle b/build.gradle
index b274652e105..439b6d6105c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -281,14 +281,12 @@ if (project.hasProperty("release.main")) {
     jreleaser {
         dryrun = false
 
-        // Used for creating a tagged release, uploading files and generating changelog.
-        // In the future we can set this up to push release tags to GitHub, but for now it's
-        // set up to do nothing.
-        // https://jreleaser.org/guide/latest/reference/release/index.html
+        // Used for creating and pushing the version tag, but this configuration ensures that
+        // an actual GitHub release isn't created (since the CLI release does that)
         release {
-            generic {
-                enabled = true
+            github {
                 skipRelease = true
+                tagName = '{{projectVersion}}'
             }
         }
 
diff --git a/smithy-cli/build.gradle b/smithy-cli/build.gradle
index 1bfc99622fe..4f18e916505 100644
--- a/smithy-cli/build.gradle
+++ b/smithy-cli/build.gradle
@@ -285,6 +285,8 @@ if (project.hasProperty("release.cli")) {
             github {
                 overwrite = true
                 tagName = '{{projectVersion}}'
+                skipTag = true
+                releaseName = 'Smithy CLI v{{{projectVersion}}}'
                 changelog {
                     // For now, we won't have a changelog added to the release. In the future, we could create a changelog-snippet
                     // from the real changelog as part of a command hook prior to the release step