Skip to content

Commit

Permalink
replace selecting version for enabling multipart
Browse files Browse the repository at this point in the history
  • Loading branch information
nickpetrovic committed Jan 9, 2025
1 parent f6afe5f commit a2dcea6
Show file tree
Hide file tree
Showing 4 changed files with 28 additions and 17 deletions.
5 changes: 2 additions & 3 deletions pkg/abstractions/volume/multipart.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,8 @@ func joinCleanPath(parts ...string) string {

func (s *GlobalVolumeService) GetFileServiceInfo(ctx context.Context, in *pb.GetFileServiceInfoRequest) (*pb.GetFileServiceInfoResponse, error) {
return &pb.GetFileServiceInfoResponse{
Ok: true,
Enabled: s.config.EndpointURL != "" && s.config.BucketName != "",
CommandVersion: s.config.CommandVersion,
Ok: true,
Enabled: s.config.Enabled && s.config.EndpointURL != "" && s.config.BucketName != "",
}, nil
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/common/config.default.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,12 @@ gateway:
maxReplicas: 10
maxGpuCount: 2
fileService:
enabled: true
endpointUrl: http://juicefs-s3-gateway.beta9.svc.cluster.local:9900
bucketName: beta9-fs
accessKey: test
secretKey: test-test-test
region:
commandVersion: 1
imageService:
localCacheEnabled: true
registryStore: local
Expand Down
5 changes: 3 additions & 2 deletions pkg/types/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,9 @@ type FileServiceConfig struct {
SecretKey string `key:"secretKey" json:"secret_key"`
Region string `key:"region" json:"region"`

// Determines which version of the copy command to use in the CLI
CommandVersion uint32 `key:"commandVersion" json:"command_version"`
// Determines if the SDK should use this service
// Requires that EndpointURL and BucketName are set
Enabled bool `key:"enabled" json:"enabled"`
}

type ImageServiceConfig struct {
Expand Down
33 changes: 22 additions & 11 deletions sdk/src/beta9/cli/volume.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import glob
from pathlib import Path
from typing import Iterable, List, Union
from typing import Iterable, List, Optional, Union

import click
from rich.table import Column, Table, box
Expand Down Expand Up @@ -151,9 +151,8 @@ def read_with_progress(
@common.command(
help="Upload or download contents to or from a volume.",
epilog="""
Version 1:
Version 1 (Streaming uploads to the Gateway):
!!! This is deprecated. Use v2 of this command.
Uploads files or directories to a volume. Downloads are not supported.
SOURCE and DESTINATION syntax:
Expand Down Expand Up @@ -193,10 +192,15 @@ def read_with_progress(
{cli_name} cp mydir/\\[a-c\\]/data[0-1].json' myvol/data
{cli_name} cp mydir/\\?/data\\?.json myvol/sub/path
Version 2:
Version 2 (Multipart uploads/downloads via an external file service):
Upload and download files or directories between a volume and your system.
This is available if the Gateway has a file service configured and enabled.
When enabled, this command will use the external file service to upload and
download files directly only using the Gateway for initiating, completing, or
cancelling the transfer.
SOURCE and DESTINATION syntax:
SOURCE can be local or remote.
Expand Down Expand Up @@ -249,10 +253,9 @@ def read_with_progress(
required=True,
)
@click.option(
"--version",
is_eager=True,
type=click.Choice(["1", "2"]),
help="The version of this command to run. [default is determined by the server]",
"--multipart/--no-multipart",
"use_multipart",
help="Use multipart upload/download. [default: True if the server supports it, else False]",
default=None,
required=False,
)
Expand All @@ -261,12 +264,20 @@ def cp(
service: ServiceClient,
source: Union[Path, RemotePath],
destination: Union[Path, RemotePath],
version: str,
use_multipart: Optional[bool],
):
res = service.volume.get_file_service_info(GetFileServiceInfoRequest())
if version == "1" or (version is None and res.command_version == 1) or not res.enabled:
if res.enabled and (use_multipart is None or use_multipart):
use_multipart = True
elif not res.enabled and use_multipart:
terminal.warn("Multipart upload/download is not currently supported by the server.")
use_multipart = False

if not use_multipart:
if "://" in str(source) or "://" in str(destination):
raise click.BadArgumentUsage("Remote path syntax ($scheme://) is not supported in v1.")
raise click.BadArgumentUsage(
"Remote path syntax ($scheme://) is not supported without multipart transfers enabled."
)
return cp_v1(service, source, destination) # type: ignore

if isinstance(source, Path) and isinstance(destination, Path):
Expand Down

0 comments on commit a2dcea6

Please sign in to comment.