[ENH] Add load service to local dev tiltfile (#4397)

## Description of changes

Make the trace exporter to export all INFO and above traces (not just
ERROR).

Also, update load service tracing to use GRPC (to match our other
services).

## Test plan

Ran locally via `tilt up`. Verified traces appear under `chroma-load`
service in jaeger.

## Documentation Changes

n/a

Author: jasonvigil

Tiltfile

@@ -140,6 +140,21 @@ else:
     target='garbage_collector'
   )
 
+if config.tilt_subcommand == "ci":
+  custom_build(
+    'load-service',
+    'docker buildx build --load -t $EXPECTED_REF --target load_service -f ./rust/load/Dockerfile .',
+    ['./rust/', './idl/', './Cargo.toml', './Cargo.lock']
+  )
+else:
+  docker_build(
+    'load-service',
+    '.',
+    only=["rust/", "idl/", "Cargo.toml", "Cargo.lock"],
+    dockerfile='./rust/load/Dockerfile',
+    target='load_service'
+  )
+
 
 # First install the CRD
 k8s_yaml(
@@ -169,6 +184,7 @@ k8s_yaml([
   'k8s/test/grafana.yaml',
   'k8s/test/jaeger-service.yaml',
   'k8s/test/jaeger.yaml',
+  'k8s/test/load-service.yaml',
   'k8s/test/minio.yaml',
   'k8s/test/prometheus.yaml',
   'k8s/test/test-memberlist-cr.yaml',
@@ -226,6 +242,7 @@ k8s_resource('sysdb', resource_deps=['sysdb-migration-latest'], labels=["chroma"
 k8s_resource('rust-frontend-service', resource_deps=['sysdb', 'logservice', 'rust-log-service'], labels=["chroma"], port_forwards='3000:8000')
 k8s_resource('query-service', resource_deps=['sysdb'], labels=["chroma"], port_forwards='50053:50051')
 k8s_resource('compaction-service', resource_deps=['sysdb'], labels=["chroma"])
+k8s_resource('load-service', resource_deps=['k8s_setup'], labels=["infrastructure"], port_forwards='3001:3001')
 k8s_resource('jaeger', resource_deps=['k8s_setup'], labels=["observability"])
 k8s_resource('grafana', resource_deps=['k8s_setup'], labels=["observability"])
 k8s_resource('prometheus', resource_deps=['k8s_setup'], labels=["observability"])

k8s/test/load-service.yaml

@@ -0,0 +1,81 @@
+---
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: chroma-load
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: load-service
+  namespace: chroma-load
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: load-service
+  template:
+    metadata:
+      namespace: chroma-load
+      labels:
+        app: load-service
+    spec:
+      containers:
+        - name: load-service
+          image: load-service
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 3001
+              name: http
+          env:
+            - name: CONFIG_PATH
+              value: /config/config.yaml
+            - name: RUST_LOG
+              value: "info"
+          volumeMounts:
+            - name: load-service-config
+              mountPath: /config/
+            - name: load-service-persistent-state
+              mountPath: /state/
+      volumes:
+        - name: load-service-config
+          configMap:
+            name: load-service-config
+        - name: load-service-persistent-state
+          hostPath:
+            path: /tmp/chroma/load-service
+            type: DirectoryOrCreate
+
+---
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: load-service-config
+  namespace: chroma-load
+data:
+  config.yaml: |
+    load_service:
+      service_name: "chroma-load"
+      otel_endpoint: "http://otel-collector.chroma:4317"
+      port: 3001
+      persistent_state_path: "/state/load-service.json"
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: load-service
+  namespace: chroma-load
+spec:
+  ports:
+    - name: http
+      port: 3001
+      targetPort: 3001
+  selector:
+    app: load-service
+  type: ClusterIP

rust/load/src/opentelemetry_config.rs

@@ -57,7 +57,7 @@ pub(crate) fn init_otel_tracing(service_name: &String, otel_endpoint: &String) {
     // Layer for adding our configured tracer.
     // Export everything at this layer. The backend i.e. honeycomb or jaeger will filter at its end.
     let exporter_layer = tracing_opentelemetry::OpenTelemetryLayer::new(tracer)
-        .with_filter(tracing_subscriber::filter::LevelFilter::ERROR);
+        .with_filter(tracing_subscriber::filter::LevelFilter::INFO);
     // Layer for printing spans to stdout. Only print INFO logs by default.
     let stdout_layer =
         BunyanFormattingLayer::new(service_name.clone().to_string(), std::io::stdout)