diff --git a/.github/workflows/update_test_file_ratings.yml b/.github/workflows/update_test_file_ratings.yml
index ce38de3614..4979551faf 100644
--- a/.github/workflows/update_test_file_ratings.yml
+++ b/.github/workflows/update_test_file_ratings.yml
@@ -37,16 +37,16 @@ jobs:
 
       - name: Get merge base info
         run: |
-          python3 test-infra/torchci/scripts/td/get_merge_base_info.py
+          python3 test-infra/tools/torchci/torchci/td/get_merge_base_info.py
         env:
           ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }}
 
       - name: Generate file test ratings
         run: |
-          python3 test-infra/torchci/scripts/td/calculate_file_test_rating.py
-          python3 test-infra/torchci/scripts/td/td_heuristic_historical_edited_files.py
+          python3 test-infra/tools/torchci/td/calculate_file_test_rating.py
+          python3 test-infra/tools/torchci/td/td_heuristic_historical_edited_files.py
           # Do not run this one, it won't change
-          # python3 test-infra/torchci/scripts/td/td_heuristic_profiling.py
+          # python3 test-infra/tools/torchci/td/td_heuristic_profiling.py
 
         env:
           ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }}
diff --git a/tools/torchci/README.md b/tools/torchci/README.md
new file mode 100644
index 0000000000..c860e54f91
--- /dev/null
+++ b/tools/torchci/README.md
@@ -0,0 +1,13 @@
+This is meant to contain all the random python scripts we make for things like
+TD, alerting, revert tracking, etc.
+
+It was originally located in torchci/scripts but moved to here to separate the
+python from the javascript/typescript.
+
+To run these files without needing to modify `sys.path` in each file, either
+1. Run `pip install -e .` from within `tools/torchci`.  The `-e` is important.
+2. Add to your `PYTHONPATH` env var via `export PYTHONPATH="${PYTHONPATH}:<repo root>/tools/torchci"`.
+3. Run every file as a module, ex `cd tools/torchci && python -m torchci.td.td_rockset_analysis`.
+
+The tests folder should contain all test related files and mirror the torchci
+folder.