diff --git a/third_party/kenlm/CMakeLists.txt b/third_party/kenlm/CMakeLists.txt
index 00aacf27db..e26e0451ea 100644
--- a/third_party/kenlm/CMakeLists.txt
+++ b/third_party/kenlm/CMakeLists.txt
@@ -17,19 +17,11 @@ set(
   submodule/util/integer_to_string.cc
   submodule/util/mmap.cc
   submodule/util/murmur_hash.cc
-  submodule/util/parallel_read.cc
   submodule/util/pool.cc
   submodule/util/read_compressed.cc
   submodule/util/scoped.cc
   submodule/util/spaces.cc
-  submodule/util/stream/chain.cc
-  submodule/util/stream/count_records.cc
-  submodule/util/stream/io.cc
-  submodule/util/stream/line_input.cc
-  submodule/util/stream/multi_progress.cc
-  submodule/util/stream/rewindable_stream.cc
   submodule/util/string_piece.cc
-  submodule/util/usage.cc
   )
 
 set(
@@ -43,7 +35,6 @@ set(
   submodule/lm/read_arpa.cc
   submodule/lm/search_hashed.cc
   submodule/lm/search_trie.cc
-  submodule/lm/sizes.cc
   submodule/lm/trie.cc
   submodule/lm/trie_sort.cc
   submodule/lm/value_build.cc
diff --git a/torchaudio/csrc/CMakeLists.txt b/torchaudio/csrc/CMakeLists.txt
index 9c58b08b43..9acdc0fe07 100644
--- a/torchaudio/csrc/CMakeLists.txt
+++ b/torchaudio/csrc/CMakeLists.txt
@@ -133,25 +133,14 @@ define_library(
 if (BUILD_CTC_DECODER)
   set(
     LIBTORCHAUDIO_DECODER_SOURCES
-    decoder/src/decoder/Decoder.h
     decoder/src/decoder/LexiconDecoder.cpp
-    decoder/src/decoder/LexiconDecoder.h
     decoder/src/decoder/Trie.cpp
-    decoder/src/decoder/Trie.h
     decoder/src/decoder/Utils.cpp
-    decoder/src/decoder/Utils.h
     decoder/src/decoder/lm/KenLM.cpp
-    decoder/src/decoder/lm/KenLM.h
-    decoder/src/decoder/lm/LM.h
-    decoder/src/dictionary/Defines.h
     decoder/src/dictionary/Dictionary.cpp
-    decoder/src/dictionary/Dictionary.h
     decoder/src/dictionary/String.cpp
-    decoder/src/dictionary/String.h
     decoder/src/dictionary/System.cpp
-    decoder/src/dictionary/System.h
     decoder/src/dictionary/Utils.cpp
-    decoder/src/dictionary/Utils.h
     )
   set(
     LIBTORCHAUDIO_DECODER_DEFINITIONS