Make stores and schedulers lists of named specs (#1496)

To migrate, change your config like this (similar for schedulers): ``` // Old: "stores": { "SOMESTORE": { "memory": {} } } // New: "stores": [ { "name": "SOMESTORE", "memory": {} } ] ``` For now this is not a breaking change, but a backwards compatible one. The backwards-compatibility code will be removed after the next release in a breaking change. Closes #834
TraceMachina · Jan 29, 2025 · c99dca6 · c99dca6
1 parent a67d4bd
commit c99dca6
Show file tree

Hide file tree

Showing 16 changed files with 303 additions and 87 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/deployment-examples/docker-compose/local-storage-cas.json5 b/deployment-examples/docker-compose/local-storage-cas.json5
@@ -4,8 +4,9 @@
 // so objects are compressed, deduplicated and uses some in-memory
 // optimizations for certain hot paths.
 {
-  "stores": {
-    "CAS_MAIN_STORE": {
+  "stores": [
+    {
+      "name": "CAS_MAIN_STORE",
       "compression": {
         "compression_algorithm": {
           "lz4": {}
@@ -21,8 +22,8 @@
           }
         }
       }
-    },
-    "AC_MAIN_STORE": {
+    }, {
+      "name": "AC_MAIN_STORE",
       "filesystem": {
         "content_path": "~/.cache/nativelink/content_path-ac",
         "temp_path": "~/.cache/nativelink/tmp_path-ac",
@@ -32,7 +33,7 @@
         }
       }
     }
-  },
+  ],
   "servers": [{
     "listener": {
       "http": {

diff --git a/deployment-examples/docker-compose/scheduler.json5 b/deployment-examples/docker-compose/scheduler.json5
@@ -1,6 +1,7 @@
 {
-  "stores": {
-    "GRPC_LOCAL_STORE": {
+  "stores": [
+    {
+      "name": "GRPC_LOCAL_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "",
@@ -9,8 +10,8 @@
         ],
         "store_type": "cas"
       }
-    },
-    "GRPC_LOCAL_AC_STORE": {
+    }, {
+      "name": "GRPC_LOCAL_AC_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "",
@@ -20,9 +21,10 @@
         "store_type": "ac"
       }
     }
-  },
-  "schedulers": {
-    "MAIN_SCHEDULER": {
+  ],
+  "schedulers": [
+    {
+      "name": "MAIN_SCHEDULER",
       "simple": {
         "supported_platform_properties": {
           "cpu_count": "minimum",
@@ -32,7 +34,7 @@
         }
       }
     }
-  },
+  ],
   "servers": [{
     "listener": {
       "http": {

diff --git a/deployment-examples/docker-compose/worker.json5 b/deployment-examples/docker-compose/worker.json5
@@ -1,6 +1,7 @@
 {
-  "stores": {
-    "GRPC_LOCAL_STORE": {
+  "stores": [
+    {
+      "name": "GRPC_LOCAL_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "",
@@ -9,8 +10,8 @@
         ],
         "store_type": "cas"
       }
-    },
-    "GRPC_LOCAL_AC_STORE": {
+    }, {
+      "name": "GRPC_LOCAL_AC_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "",
@@ -19,8 +20,8 @@
         ],
         "store_type": "ac"
       }
-    },
-    "WORKER_FAST_SLOW_STORE": {
+    }, {
+      "name": "WORKER_FAST_SLOW_STORE",
       "fast_slow": {
         "fast": {
           "filesystem": {
@@ -39,7 +40,7 @@
         }
       }
     }
-  },
+  ],
   "workers": [{
     "local": {
       "worker_api_endpoint": {

diff --git a/kubernetes/components/worker/worker.json5 b/kubernetes/components/worker/worker.json5
@@ -1,6 +1,7 @@
 {
-  "stores": {
-    "GRPC_LOCAL_STORE": {
+  "stores": [
+    {
+      "name": "GRPC_LOCAL_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "",
@@ -9,8 +10,8 @@
         ],
         "store_type": "cas"
       }
-    },
-    "GRPC_LOCAL_AC_STORE": {
+    }, {
+      "name": "GRPC_LOCAL_AC_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "",
@@ -19,8 +20,8 @@
         ],
         "store_type": "ac"
       }
-    },
-    "WORKER_FAST_SLOW_STORE": {
+    }, {
+      "name": "WORKER_FAST_SLOW_STORE",
       "fast_slow": {
         "fast": {
           "filesystem": {
@@ -38,7 +39,7 @@
         }
       }
     }
-  },
+  ],
   "workers": [{
     "local": {
       "worker_api_endpoint": {

diff --git a/kubernetes/nativelink/nativelink-config.json5 b/kubernetes/nativelink/nativelink-config.json5
@@ -2,8 +2,9 @@
 // `~/.cache/nativelink`. When this location is mounted as a PersistentVolume
 // it persists the cache across restarts.
 {
-  "stores": {
-    "CAS_MAIN_STORE": {
+  "stores": [
+    {
+      "name": "CAS_MAIN_STORE",
       "existence_cache": {
         "backend": {
           "compression": {
@@ -22,8 +23,8 @@
           }
         }
       }
-    },
-    "AC_MAIN_STORE": {
+    }, {
+      "name": "AC_MAIN_STORE",
       "completeness_checking": {
         "backend": {
           "filesystem": {
@@ -41,9 +42,10 @@
         }
       }
     }
-  },
-  "schedulers": {
-    "MAIN_SCHEDULER": {
+  ],
+  "schedulers": [
+    {
+      "name": "MAIN_SCHEDULER",
       // TODO(adams): use the right scheduler because reclient doesn't use the cached results?
       // TODO(adams): max_bytes_per_stream
       "simple": {
@@ -68,7 +70,7 @@
         }
       }
     }
-  },
+  ],
   "servers": [{
     // Only publish metrics on a private port.
     "listener": {

diff --git a/nativelink-config/BUILD.bazel b/nativelink-config/BUILD.bazel
@@ -31,6 +31,7 @@ rust_test_suite(
     name = "integration",
     timeout = "short",
     srcs = [
+        "tests/backwards_compat_test.rs",
         "tests/deserialization_test.rs",
     ],
     deps = [
@@ -40,6 +41,7 @@ rust_test_suite(
         "@crates//:humantime",
         "@crates//:pretty_assertions",
         "@crates//:serde",
+        "@crates//:serde_json",
         "@crates//:serde_json5",
     ],
 )

diff --git a/nativelink-config/Cargo.toml b/nativelink-config/Cargo.toml
@@ -12,3 +12,4 @@ shellexpand = { version = "3.1.0", default-features = false, features = ["base-0
 
 [dev-dependencies]
 pretty_assertions = { version = "1.4.1", features = ["std"] }
+serde_json = { version = "1.0.138", default-features = false }
diff --git a/nativelink-config/examples/basic_cas.json5 b/nativelink-config/examples/basic_cas.json5
@@ -1,6 +1,7 @@
 {
-  "stores": {
-    "AC_MAIN_STORE": {
+  "stores": [
+    {
+      "name": "AC_MAIN_STORE",
       "filesystem": {
         "content_path": "/tmp/nativelink/data-worker-test/content_path-ac",
         "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac",
@@ -9,8 +10,8 @@
           "max_bytes": 1000000000
         }
       }
-    },
-    "WORKER_FAST_SLOW_STORE": {
+    }, {
+      "name": "WORKER_FAST_SLOW_STORE",
       "fast_slow": {
         // "fast" must be a "filesystem" store because the worker uses it to make
         // hardlinks on disk to a directory where the jobs are running.
@@ -34,9 +35,10 @@
         }
       }
     }
-  },
-  "schedulers": {
-    "MAIN_SCHEDULER": {
+  ],
+  "schedulers": [
+    {
+      "name": "MAIN_SCHEDULER",
       "simple": {
         "supported_platform_properties": {
           "cpu_count": "minimum",
@@ -59,7 +61,7 @@
         }
       }
     }
-  },
+  ],
   "workers": [{
     "local": {
       "worker_api_endpoint": {

diff --git a/nativelink-config/examples/filesystem_cas.json5 b/nativelink-config/examples/filesystem_cas.json5
@@ -4,8 +4,9 @@
 // so objects are compressed, deduplicated and uses some in-memory
 // optimizations for certain hot paths.
 {
-  "stores": {
-    "FS_CONTENT_STORE": {
+  "stores": [
+    {
+      "name": "FS_CONTENT_STORE",
       "compression": {
         "compression_algorithm": {
           "lz4": {}
@@ -21,8 +22,8 @@
           }
         }
       }
-    },
-    "CAS_MAIN_STORE": {
+    }, {
+      "name": "CAS_MAIN_STORE",
       "verify": {
         "backend": {
           // Because we are using a dedup store, we can bypass small objects
@@ -78,8 +79,8 @@
         "verify_size": true,
         "verify_hash": true
       }
-    },
-    "AC_MAIN_STORE": {
+    }, {
+      "name": "AC_MAIN_STORE",
       "filesystem": {
         "content_path": "/tmp/nativelink/data/content_path-ac",
         "temp_path": "/tmp/nativelink/data/tmp_path-ac",
@@ -89,9 +90,10 @@
         }
       }
     }
-  },
-  "schedulers": {
-    "MAIN_SCHEDULER": {
+  ],
+  "schedulers": [
+    {
+      "name": "MAIN_SCHEDULER",
       "simple": {
         "supported_platform_properties": {
           "cpu_count": "minimum",
@@ -113,7 +115,7 @@
         }
       }
     }
-  },
+  ],
   "servers": [{
     "listener": {
       "http": {

diff --git a/nativelink-config/examples/redis.json b/nativelink-config/examples/redis.json
@@ -1,18 +1,19 @@
 {
-  "stores": {
-    "CAS_FAST_SLOW_STORE": {
+  "stores": [
+    {
+      "name": "CAS_FAST_SLOW_STORE",
       "redis_store": {
         "addresses": ["redis://127.0.0.1:6379/"],
         "mode": "cluster"
       }
-    },
-    "AC_FAST_SLOW_STORE": {
+    }, {
+      "name": "AC_FAST_SLOW_STORE",
       "redis_store": {
         "addresses": ["redis://127.0.0.1:6379/"],
         "mode": "cluster"
       }
-    },
-    "AC_MAIN_STORE": {
+    }, {
+      "name": "AC_MAIN_STORE",
       "completeness_checking": {
         "backend": {
           "ref_store": {
@@ -25,8 +26,8 @@
           }
         }
       }
-    },
-    "CAS_MAIN_STORE": {
+    }, {
+      "name": "CAS_MAIN_STORE",
       "existence_cache": {
         "backend": {
           "compression": {
@@ -42,7 +43,7 @@
         }
       }
     }
-  },
+  ],
   "servers": [
     {
       "listener": {
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,3 +12,4 @@ shellexpand = { version = "3.1.0", default-features = false, features = ["base-0

		[dev-dependencies]
		pretty_assertions = { version = "1.4.1", features = ["std"] }
		serde_json = { version = "1.0.138", default-features = false }