Rerun eval, update timings, and enable CUDA by default (#139)

This reruns evaluation (to sync with code and update pipeline), and adds timings on a new gaming rig to our timing chart. It also introduces a code change to `config.py`: if CUDA is available, it is now enabled by default, so `pixi run -e eval-cuda dvc repro` is enough to do CUDA-powered repro runs.
CCRI-POPROX · Dec 4, 2024 · 8c95b7a · 8c95b7a
1 parent 4fef134
commit 8c95b7a
Show file tree

Hide file tree

Showing 10 changed files with 424 additions and 416 deletions.
diff --git a/README.md b/README.md
@@ -243,28 +243,27 @@ $ curl -X POST -H "Content-Type: application/json" -d @tests/request_data/basic-
 The default setup for this package is CPU-only, which works for basic testing
 and for deployment, but is very inefficient for evaluation.  The current set of
 models work on both CUDA (on Linux with NVidia cards) and MPS (macOS on Apple
-Silicon).  To make use of a GPU, do the following:
-
-1.  Set the `POPROX_REC_DEVICE` environment variable to `cuda` or `mps`.
-
-2.  Run `dvc repro` under the `eval` or `dev` environment (using either `pixi
-    run` or `pixi shell`).
+Silicon).  To make use of a GPU, do the following:, run `dvc repro` under the
+`eval-cuda` or `dev-cuda` environment (using either `pixi run` or `pixi shell`).
 
 Timing information for generating recommendations with the MIND validation set:
 
-| Machine | CPU              | GPU        | Rec. Time | Rec. Power | Eval Time |
-| ------- | :--------------: | :--------: | :-------: | :--------: | :-------: |
-| [DXC][] | EPYC 7662 (2GHz) | A40 (CUDA) | 45m¹      | 418.5 Wh   | 24m       |
-| [MBP][] | Apple M2 Pro     | -          | <20hr²    |            | 30m²      |
-| [MBP][] | Apple M2 Pro     | M2 (MPS)   | <12hr²    |            |           |
+| Machine      | CPU                | GPU         | Rec. Time | Rec. Power | Eval Time |
+| ------------ | :----------------: | :---------: | :-------: | :--------: | :-------: |
+| [Cruncher][] | EPYC 7662 (2GHz)   | A40 (CUDA)  | 45m¹      | 418.5 Wh   | 24m       |
+| [Screamer][] | i9 14900K (3.2GHz) | 4090 (CUDA) | 28m16s²   |            | 14m       |
+| [Ranger][]   | Apple M2 Pro       | -           | <20hr³    |            | 30m³      |
+| [Ranger][]   | Apple M2 Pro       | M2 (MPS)    | <12hr³    |            |           |
 
-[DXC]: https://codex.lenskit.org/hardware/cruncher.html
-[MBP]: https://codex.lenskit.org/hardware/mbp2.html
+[Cruncher]: https://codex.lenskit.org/hardware/cruncher.html
+[Screamer]: https://codex.lenskit.org/hardware/screamer.html
+[Ranger]: https://codex.lenskit.org/hardware/ranger.html
 
 Footnotes:
 
 1. Using 12 worker processes
-2. Estimated based on early progress, not run to completion.
+2. Using 8 worker processes
+3. Estimated based on early progress, not run to completion.
 
 ## Editor Setup
 

diff --git a/dvc.lock b/dvc.lock
@@ -9,22 +9,22 @@ stages:
       size: 103456245
     - path: outputs/mind-val/recommendations
       hash: md5
-      md5: edebe062ad957f0e991f4abe1eafa834.dir
-      size: 44827588
-      nfiles: 4
+      md5: 7d1cad4d5fffb9e47e6abb62d2d27f16.dir
+      size: 44878647
+      nfiles: 8
     - path: src/poprox_recommender/evaluation/evaluate.py
       hash: md5
       md5: 63f8d61ecc6708c41f542eba397c0d59
       size: 4608
     outs:
     - path: outputs/mind-val/metrics.csv
       hash: md5
-      md5: 316cf979567f09d888c9131dab7a51d8
-      size: 673
+      md5: 4885ed026f007b16fb6a39c48f9a9e68
+      size: 670
     - path: outputs/mind-val/profile-metrics.csv.gz
       hash: md5
-      md5: 1ee7d49c679c1508a3122a383f21fe83
-      size: 34022756
+      md5: 40e34937389500c9ce74cb3d93d215df
+      size: 34000151
   recommend-mind-val:
     cmd: python -m poprox_recommender.evaluation.generate -M MINDlarge_dev -o outputs/mind-val
     deps:
@@ -34,19 +34,19 @@ stages:
       size: 103456245
     - path: src/poprox_recommender/evaluation/generate/
       hash: md5
-      md5: a4149a7d507e1aef68a499b39fb6dff2.dir
-      size: 31855
+      md5: 5a40f035067811292117bdb23065a02a.dir
+      size: 31867
       nfiles: 8
     outs:
     - path: outputs/mind-val/embeddings.parquet
       hash: md5
-      md5: b11eef8a09ca9d7b5f734da1739228a3
-      size: 20625907
+      md5: 1de40da0c2aed9255997c20239c55f3f
+      size: 20621749
     - path: outputs/mind-val/recommendations
       hash: md5
-      md5: edebe062ad957f0e991f4abe1eafa834.dir
-      size: 44827588
-      nfiles: 4
+      md5: 7d1cad4d5fffb9e47e6abb62d2d27f16.dir
+      size: 44878647
+      nfiles: 8
   recommend-mind-small:
     cmd: python -m poprox_recommender.evaluation.generate -M MINDsmall_dev -o outputs/mind-small
     deps:
@@ -56,19 +56,19 @@ stages:
       size: 30948560
     - path: src/poprox_recommender/evaluation/generate/
       hash: md5
-      md5: a4149a7d507e1aef68a499b39fb6dff2.dir
-      size: 31855
+      md5: 5a40f035067811292117bdb23065a02a.dir
+      size: 31867
       nfiles: 8
     outs:
     - path: outputs/mind-small/embeddings.parquet
       hash: md5
-      md5: 5eeb31cc486e7325283880285d35813a
-      size: 15967246
+      md5: 153c2f32568ac61f1c07b6cdb0f951ba
+      size: 15967347
     - path: outputs/mind-small/recommendations
       hash: md5
-      md5: af4d9f11e686b8ef139b915d9021b1f3.dir
-      size: 8910177
-      nfiles: 4
+      md5: e3268d2d750b0b5f3e6c5afa5ba13453.dir
+      size: 9111609
+      nfiles: 8
   measure-mind-small:
     cmd: python -m poprox_recommender.evaluation.evaluate -M MINDsmall_dev mind-small
     deps:
@@ -78,22 +78,22 @@ stages:
       size: 103456245
     - path: outputs/mind-small/recommendations
       hash: md5
-      md5: af4d9f11e686b8ef139b915d9021b1f3.dir
-      size: 8910177
-      nfiles: 4
+      md5: e3268d2d750b0b5f3e6c5afa5ba13453.dir
+      size: 9111609
+      nfiles: 8
     - path: src/poprox_recommender/evaluation/evaluate.py
       hash: md5
       md5: 63f8d61ecc6708c41f542eba397c0d59
       size: 4608
     outs:
     - path: outputs/mind-small/metrics.csv
       hash: md5
-      md5: 164b955b213dfd440bb7fec08259acc1
-      size: 674
+      md5: 4823af1d567142b8f1d628f2f0961589
+      size: 671
     - path: outputs/mind-small/profile-metrics.csv.gz
       hash: md5
-      md5: 885321013d5c4d6d59093edb503104d5
-      size: 6610049
+      md5: e1ddec19dc5ff6cf886186524fbf1a4c
+      size: 6606761
   recommend-mind-subset:
     cmd: python -m poprox_recommender.evaluation.generate -M MINDsmall_dev --subset=1000
       -o outputs/mind-subset
@@ -104,19 +104,19 @@ stages:
       size: 30948560
     - path: src/poprox_recommender/evaluation/generate/
       hash: md5
-      md5: a4149a7d507e1aef68a499b39fb6dff2.dir
-      size: 31855
+      md5: 5a40f035067811292117bdb23065a02a.dir
+      size: 31867
       nfiles: 8
     outs:
     - path: outputs/mind-subset/embeddings.parquet
       hash: md5
-      md5: 03f0c7d5c0ec84fa93a807eb1f31dc43
-      size: 5113186
+      md5: bf5b727866ba25845198b5745d043d4f
+      size: 5128758
     - path: outputs/mind-subset/recommendations
       hash: md5
-      md5: 268d9cbefa48fa3392d73a2c8015d7ac.dir
-      size: 151657
-      nfiles: 4
+      md5: c57e508e552b25f9ebc71e4a695b11ac.dir
+      size: 230729
+      nfiles: 8
   measure-mind-subset:
     cmd: python -m poprox_recommender.evaluation.evaluate -M MINDsmall_dev mind-subset
     deps:
@@ -126,19 +126,19 @@ stages:
       size: 103456245
     - path: outputs/mind-subset/recommendations
       hash: md5
-      md5: 268d9cbefa48fa3392d73a2c8015d7ac.dir
-      size: 151657
-      nfiles: 4
+      md5: c57e508e552b25f9ebc71e4a695b11ac.dir
+      size: 230729
+      nfiles: 8
     - path: src/poprox_recommender/evaluation/evaluate.py
       hash: md5
       md5: 63f8d61ecc6708c41f542eba397c0d59
       size: 4608
     outs:
     - path: outputs/mind-subset/metrics.csv
       hash: md5
-      md5: 749f2d836ca23b560d8f35551bb53d7a
-      size: 671
+      md5: cf0b1f2b4d185d4d479a2b5995ee40f6
+      size: 673
     - path: outputs/mind-subset/profile-metrics.csv.gz
       hash: md5
-      md5: fb0373335e5e7c7fb3c998c0ed930f49
-      size: 90985
+      md5: 4a658459093af34e63ec8905f64b8146
+      size: 90608
diff --git a/notebooks/dvc.lock b/notebooks/dvc.lock
@@ -5,46 +5,46 @@ stages:
     deps:
     - path: ../outputs/mind-val/profile-metrics.csv.gz
       hash: md5
-      md5: 1ee7d49c679c1508a3122a383f21fe83
-      size: 34022756
+      md5: 40e34937389500c9ce74cb3d93d215df
+      size: 34000151
     - path: mind-val.md
       hash: md5
       md5: 386c3b5588f70d8a845b0faaa52c4859
       size: 3758
     outs:
     - path: mind-val.ipynb
       hash: md5
-      md5: 4f455a947e02d17f65891628a6528c75
-      size: 180580
+      md5: 82246ae5007d7196ed65601dd3999742
+      size: 180360
   results-mind-small:
     cmd: jupytext --to notebook --execute mind-small.md
     deps:
     - path: ../outputs/mind-small/profile-metrics.csv.gz
       hash: md5
-      md5: 885321013d5c4d6d59093edb503104d5
-      size: 6610049
+      md5: e1ddec19dc5ff6cf886186524fbf1a4c
+      size: 6606761
     - path: mind-small.md
       hash: md5
       md5: 170f50dd7278beff3c040fed5ded0074
       size: 3774
     outs:
     - path: mind-small.ipynb
       hash: md5
-      md5: 005446bda1bdd11ecf7774229b0eb970
-      size: 182164
+      md5: c99231a9284b9690992f4deaee20be7b
+      size: 182179
   results-mind-subset:
     cmd: jupytext --to notebook --execute mind-subset.md
     deps:
     - path: ../outputs/mind-subset/profile-metrics.csv.gz
       hash: md5
-      md5: fb0373335e5e7c7fb3c998c0ed930f49
-      size: 90985
+      md5: 4a658459093af34e63ec8905f64b8146
+      size: 90608
     - path: mind-subset.md
       hash: md5
       md5: fd56e0a68ae9edd86be8a2a0fda2b78d
       size: 3782
     outs:
     - path: mind-subset.ipynb
       hash: md5
-      md5: 092a19124e7bf3f3f0e336d75d112284
-      size: 178521
+      md5: 7e6630067ae4444c84c5f27a005a9182
+      size: 177386