llama : add missing kv clear in llama_beam_search (ggerganov#6664)

NeoZhangJianyu · Apr 14, 2024 · 1958f7e · 1958f7e
1 parent 04fbc5f
commit 1958f7e
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/llama.cpp b/llama.cpp
@@ -13063,6 +13063,11 @@ struct llama_beam_search_data {
             }
             llama_logit_info logit_info(ctx);
             std::vector<llama_token_data> next_tokens = logit_info.top_k(n_beams);
+
+            // Clear the kv slot so that other beams may try different tokens at this position. The llama_decode()
+            // call in loop() will conclusively fill in the kv slot once the beams converge at this position.
+            llama_kv_cache_seq_rm(ctx, 0, n_past, -1);
+
             size_t i=0;
             if (next_beams.size() < n_beams) {
                 for (; next_beams.size() < n_beams ; ++i) {