diff --git a/common/changes/@typespec/compiler/fix-rekeyable-map_2023-10-26-19-52.json b/common/changes/@typespec/compiler/fix-rekeyable-map_2023-10-26-19-52.json
new file mode 100644
index 0000000000..78fcaa6d25
--- /dev/null
+++ b/common/changes/@typespec/compiler/fix-rekeyable-map_2023-10-26-19-52.json
@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@typespec/compiler",
+      "comment": "[Internal] Fix: `RekeyableMap` kept track of old value if rekeying to an existing item",
+      "type": "none"
+    }
+  ],
+  "packageName": "@typespec/compiler"
+}
\ No newline at end of file
diff --git a/packages/compiler/src/core/util.ts b/packages/compiler/src/core/util.ts
index c279b8dc33..78a4aafb22 100644
--- a/packages/compiler/src/core/util.ts
+++ b/packages/compiler/src/core/util.ts
@@ -585,6 +585,10 @@ class RekeyableMapImpl<K, V> implements RekeyableMap<K, V> {
       return false;
     }
     this.#keys.delete(existingKey);
+    const newKeyItem = this.#keys.get(newKey);
+    if (newKeyItem) {
+      this.#values.delete(newKeyItem);
+    }
     keyItem.key = newKey;
     this.#keys.set(newKey, keyItem);
     return true;
diff --git a/packages/compiler/test/util.test.ts b/packages/compiler/test/util.test.ts
index efbfd3b057..0a21468ef2 100644
--- a/packages/compiler/test/util.test.ts
+++ b/packages/compiler/test/util.test.ts
@@ -72,5 +72,25 @@ describe("compiler: util", () => {
         ]
       );
     });
+
+    it("rekeying to existing key override the target", () => {
+      const map = createRekeyableMap([
+        ["a", "pos 1"],
+        ["b", "pos 2"],
+        ["c", "pos 3"],
+        ["d", "pos 4"],
+      ]);
+
+      map.rekey("c", "b");
+
+      deepStrictEqual(
+        [...map.entries()],
+        [
+          ["a", "pos 1"],
+          ["b", "pos 3"],
+          ["d", "pos 4"],
+        ]
+      );
+    });
   });
 });