add local jfs gateway, fix edge cases, dynamic chunk size

beam-cloud · Jan 9, 2025 · 425caf1 · 425caf1
1 parent 063dc31
commit 425caf1
Show file tree

Hide file tree

Showing 14 changed files with 641 additions and 292 deletions.
diff --git a/hack/k3d.yaml b/hack/k3d.yaml
@@ -19,6 +19,9 @@ ports:
 - port: 8008:8008
   nodeFilters:
   - loadbalancer
+- port: 9900:9900
+  nodeFilters:
+  - loadbalancer
 
 volumes:
 - volume: $PWD/manifests/k3d:k3s-manifests-custom

diff --git a/manifests/kustomize/components/juicefs/helm-chart-generator.yaml b/manifests/kustomize/components/juicefs/helm-chart-generator.yaml
@@ -19,3 +19,39 @@ valuesInline:
   master:
     persistence:
       size: 1Gi
+---
+apiVersion: builtin
+kind: HelmChartInflationGenerator
+metadata:
+  name: &name juicefs-s3-gateway
+releaseName: *name
+name: &name juicefs-s3-gateway
+repo: https://juicedata.github.io/charts/
+version: 0.11.2
+namespace: beta9
+includeCRDs: true
+skipTests: true
+valuesInline:
+  nameOverride: *name
+  fullnameOverride: *name
+  port: 9900
+  service:
+    type: LoadBalancer
+  secret:
+    name: beta9-fs
+    metaurl: redis://juicefs-redis-master:6379/0
+    storage: s3
+    accessKey: test
+    secretKey: test-test-test
+    bucket: http://localstack:4566/juicefs
+
+  # -- JuiceFS format options. Separated by spaces
+  # Example: "--inodes=1000000 --block-size=4M"
+  # Ref: https://juicefs.com/docs/community/command_reference#format
+  formatOptions: "--block-size=4096 --no-update"
+
+  # -- Gateway Options. Separated by spaces
+  # Example: "--get-timeout=60 --put-timeout=60"
+  # CE Ref: https://juicefs.com/docs/community/command_reference#gateway
+  # EE Ref: https://juicefs.com/docs/cloud/reference/command_reference/#gateway
+  options: "--buffer-size=300 --cache-size=0 --prefetch=-1 --no-usage-report"
diff --git a/manifests/kustomize/components/juicefs/kustomization.yaml b/manifests/kustomize/components/juicefs/kustomization.yaml
@@ -4,3 +4,14 @@ kind: Component
 
 generators:
 - helm-chart-generator.yaml
+
+patches:
+- target:
+    kind: Deployment
+    name: juicefs-s3-gateway
+  patch: |-
+    - op: remove
+      path: /spec/template/spec/containers/0/env/0/valueFrom
+    - op: add
+      path: /spec/template/spec/containers/0/env/0/value
+      value: 0.0.0.0
diff --git a/manifests/kustomize/overlays/cluster-dev/kustomization.yaml b/manifests/kustomize/overlays/cluster-dev/kustomization.yaml
@@ -61,3 +61,11 @@ patches:
     - op: replace
       path: /spec/replicas
       value: 0
+# Replace RollingUpdate with Recreate for all Deployments in this overlay
+- target:
+    kind: Deployment
+  patch: |
+    - op: add
+      path: /spec/strategy
+      value:
+        type: Recreate
diff --git a/pkg/abstractions/volume/volume.go b/pkg/abstractions/volume/volume.go
@@ -228,6 +228,36 @@ func (vs *GlobalVolumeService) MovePath(ctx context.Context, in *pb.MovePathRequ
 	}, nil
 }
 
+func (vs *GlobalVolumeService) StatPath(ctx context.Context, in *pb.StatPathRequest) (*pb.StatPathResponse, error) {
+	authInfo, _ := auth.AuthInfoFromContext(ctx)
+
+	path, err := vs.getFilePath(ctx, in.Path, authInfo.Workspace)
+	if err != nil {
+		return &pb.StatPathResponse{
+			Ok:     false,
+			ErrMsg: err.Error(),
+		}, nil
+	}
+
+	info, err := os.Stat(path)
+	if err != nil {
+		return &pb.StatPathResponse{
+			Ok:     true,
+			ErrMsg: "Path does not exist",
+		}, nil
+	}
+
+	return &pb.StatPathResponse{
+		Ok: true,
+		PathInfo: &pb.PathInfo{
+			Path:    in.Path,
+			Size:    uint64(info.Size()),
+			ModTime: timestamppb.New(info.ModTime()),
+			IsDir:   info.IsDir(),
+		},
+	}, nil
+}
+
 // Volume business logic
 func (vs *GlobalVolumeService) getOrCreateVolume(ctx context.Context, workspace *types.Workspace, volumeName string) (*types.Volume, error) {
 	volume, err := vs.backendRepo.GetOrCreateVolume(ctx, workspace.Id, volumeName)

diff --git a/pkg/abstractions/volume/volume.proto b/pkg/abstractions/volume/volume.proto
@@ -13,6 +13,7 @@ service VolumeService {
   rpc DeletePath(DeletePathRequest) returns (DeletePathResponse) {}
   rpc CopyPathStream(stream CopyPathRequest) returns (CopyPathResponse) {}
   rpc MovePath(MovePathRequest) returns (MovePathResponse) {}
+  rpc StatPath(StatPathRequest) returns (StatPathResponse) {}
 
   // Multipart Upload
   rpc CreatePresignedURL(CreatePresignedURLRequest) returns (CreatePresignedURLResponse) {}
@@ -107,6 +108,15 @@ message MovePathResponse {
   string new_path = 3;
 }
 
+message StatPathRequest {
+  string path = 1;
+}
+
+message StatPathResponse {
+  bool ok = 1;
+  string err_msg = 2;
+  PathInfo path_info = 3;
+}
 
 message PresignedURLParams {
   string upload_id = 1;

diff --git a/pkg/common/config.default.yaml b/pkg/common/config.default.yaml
@@ -57,10 +57,10 @@ gateway:
     maxReplicas: 10
     maxGpuCount: 2
 fileService:
-  endpointUrl: https://data-stage-beam-cloud.fly.dev
+  endpointUrl: http://juicefs-s3-gateway.beta9.svc.cluster.local:9900
   bucketName: beta9-fs
-  accessKey:
-  secretKey:
+  accessKey: test
+  secretKey: test-test-test
   region:
 imageService:
   localCacheEnabled: true