Skip to content

Commit

Permalink
update tools to support storage class
Browse files Browse the repository at this point in the history 
Signed-off-by: tianpingan <[email protected]>
  • Loading branch information
@Tianpingan
Tianpingan committed Dec 25, 2023
1 parent bfabdea commit e0eb79f
Show file tree
Hide file tree
Showing 8 changed files with 73 additions and 22 deletions.
1 change: 1 addition & 0 deletions curvefs/proto/common.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ message CopysetInfo {
}

enum StorageClass {
NOT_SET = 0;
STANDARD = 1;
REDUCED_REDUNDANCY = 2;
STANDARD_IA = 3;
Expand Down
11 changes: 11 additions & 0 deletions curvefs/src/tools/create/curvefs_create_fs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ DECLARE_string(s3_bucket_name);
DECLARE_uint64(s3_blocksize);
DECLARE_uint64(s3_chunksize);
DECLARE_uint32(s3_objectPrefix);
DECLARE_string(s3_storageClass);
DECLARE_uint32(rpcTimeoutMs);
DECLARE_uint32(rpcRetryTimes);
DECLARE_bool(enableSumInDir);
Expand Down Expand Up @@ -100,6 +101,7 @@ void CreateFsTool::PrintHelp() {
<< " -s3_blocksize=" << FLAGS_s3_blocksize
<< " -s3_chunksize=" << FLAGS_s3_chunksize
<< " -s3_objectPrefix=" << FLAGS_s3_objectPrefix
<< " -s3_storageClass=NOT_SET|STANDARD|REDUCED_REDUNDANCY|STANDARD_IA|ONEZONE_IA|INTELLIGENT_TIERING|GLACIER|DEEP_ARCHIVE"
<< "]\n[-fsType=hybrid -volumeBlockGroupSize="
<< FLAGS_volumeBlockGroupSize
<< " -volumeBlockSize=" << FLAGS_volumeBlockSize
Expand All @@ -113,6 +115,7 @@ void CreateFsTool::PrintHelp() {
<< " -s3_blocksize=" << FLAGS_s3_blocksize
<< " -s3_chunksize=" << FLAGS_s3_chunksize
<< " -s3_objectPrefix=" << FLAGS_s3_objectPrefix
<< " -s3_storageClass=NOT_SET|STANDARD|REDUCED_REDUNDANCY|STANDARD_IA|ONEZONE_IA|INTELLIGENT_TIERING|GLACIER|DEEP_ARCHIVE"
<< "]" << std::endl;
}

Expand All @@ -134,6 +137,7 @@ void CreateFsTool::AddUpdateFlags() {
AddUpdateFlagsFunc(curvefs::tools::SetS3_blocksize);
AddUpdateFlagsFunc(curvefs::tools::SetS3_chunksize);
AddUpdateFlagsFunc(curvefs::tools::SetS3_objectPrefix);
AddUpdateFlagsFunc(curvefs::tools::SetS3_storageClass);
AddUpdateFlagsFunc(curvefs::tools::SetRpcTimeoutMs);
AddUpdateFlagsFunc(curvefs::tools::SetRpcRetryTimes);
AddUpdateFlagsFunc(curvefs::tools::SetEnableSumInDir);
Expand Down Expand Up @@ -179,6 +183,13 @@ int CreateFsTool::Init() {
s3->set_blocksize(FLAGS_s3_blocksize);
s3->set_chunksize(FLAGS_s3_chunksize);
s3->set_objectprefix(FLAGS_s3_objectPrefix);
curvefs::common::StorageClass storageClass;
if (!StorageClass_Parse(FLAGS_s3_storageClass, &storageClass)) {
std::cerr << "Parse storageClass error, only support "
"NOT_SET|STANDARD|REDUCED_REDUNDANCY|STANDARD_IA|ONEZONE_IA|INTELLIGENT_TIERING|GLACIER|DEEP_ARCHIVE";
return -1;
}
s3->set_storageclass(storageClass);
request.mutable_fsdetail()->set_allocated_s3info(s3);
return 0;
};
Expand Down
7 changes: 7 additions & 0 deletions curvefs/src/tools/curvefs_tool_define.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ DEFINE_string(s3_bucket_name, "bucketname", "s3 bucket name");
DEFINE_uint64(s3_blocksize, 1048576, "s3 block size");
DEFINE_uint64(s3_chunksize, 4194304, "s3 chunk size");
DEFINE_uint32(s3_objectPrefix, 0, "object prefix");
DEFINE_string(s3_storageClass, "NOT_SET", "s3 storage classes");
DEFINE_bool(enableSumInDir, false, "statistic info in xattr");
DEFINE_uint64(capacity, (uint64_t)0,
"capacity of fs, unit is bytes, default 0 to disable quota");
Expand Down Expand Up @@ -256,6 +257,12 @@ std::function<void(curve::common::Configuration*, google::CommandLineFlagInfo*)>
std::placeholders::_2, "s3_objectPrefix", "s3.objectPrefix",
&FLAGS_s3_objectPrefix);

std::function<void(curve::common::Configuration*, google::CommandLineFlagInfo*)>
SetS3_storageClass =
std::bind(&SetDiffFlagInfo<fLS::clstring>, std::placeholders::_1,
std::placeholders::_2, "s3_storageClass", "s3.storageClass",
&FLAGS_s3_storageClass);

std::function<void(curve::common::Configuration*, google::CommandLineFlagInfo*)>
SetEnableSumInDir = std::bind(&SetFlagInfo<bool>, std::placeholders::_1,
std::placeholders::_2, "enableSumInDir",
Expand Down
4 changes: 4 additions & 0 deletions curvefs/src/tools/curvefs_tool_define.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,10 @@ extern std::function<void(curve::common::Configuration*,
extern std::function<void(curve::common::Configuration*,
google::CommandLineFlagInfo*)>
SetS3_objectPrefix;
extern std::function<void(curve::common::Configuration*,
google::CommandLineFlagInfo*)>
SetS3_storageClass;

extern std::function<void(curve::common::Configuration*,
google::CommandLineFlagInfo*)>
SetEnableSumInDir;
Expand Down
4 changes: 3 additions & 1 deletion tools-v2/internal/error/error.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -500,7 +500,9 @@ var (
ErrInvalidMetaServerAddr = func() *CmdError {
return NewInternalCmdError(80, "invalid metaserver external addr: %s")
}

ErrUnknownStorageClass = func() *CmdError {
return NewInternalCmdError(81, "unknown storage class: %s")
}
// http error
ErrHttpUnreadableResult = func() *CmdError {
return NewHttpResultCmdError(1, "http response is unreadable, the uri is: %s, the error is: %s")
Expand Down
10 changes: 10 additions & 0 deletions tools-v2/internal/utils/proto.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,3 +223,13 @@ func ParseThrottleType(typeStr string) (nameserver2.ThrottleType, *cmderror.CmdE
}
return nameserver2.ThrottleType(throttleType), retErr
}

func TranslateStorageClass(storageclass string) (common.StorageClass, *cmderror.CmdError) {
value, found := common.StorageClass_value[storageclass]
var retErr cmderror.CmdError
if !found {
retErr = *cmderror.ErrUnknownStorageClass()
retErr.Format(storageclass)
}
return common.StorageClass(value), &retErr
}
24 changes: 15 additions & 9 deletions tools-v2/pkg/cli/command/curvefs/create/fs/fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ import (

const (
fsExample = `$ curve fs create fs --fsname test1
$ curve fs create fs --fsname test1 --fstype s3 --s3.ak AK --s3.sk SK --s3.endpoint http://localhost:9000 --s3.bucketname test1 --s3.blocksize 4MiB --s3.chunksize 4MiB
$ curve fs create fs --fsname test1 --fstype s3 --s3.ak AK --s3.sk SK --s3.endpoint http://localhost:9000 --s3.bucketname test1 --s3.blocksize 4MiB --s3.chunksize 4MiB --s3.storageclass STANDARD
$ curve fs create fs --fsname test1 --fstype volume --volume.bitmaplocation AtStart --volume.blockgroupsize 128MiB --volume.blocksize 4KiB --volume.name volume --volume.password password --volume.size 1MiB --volume.slicesize 1MiB --volume.user user
$ curve fs create fs --fsname test1 --fstype hybrid --s3.ak AK --s3.sk SK --s3.endpoint http://localhost:9000 --s3.bucketname test1 --s3.blocksize 4MiB --s3.chunksize 4MiB --volume.bitmaplocation AtStart --volume.blockgroupsize 128MiB --volume.blocksize 4KiB --volume.name volume --volume.password password --volume.size 1MiB --volume.slicesize 1MiB --volume.user user`
$ curve fs create fs --fsname test1 --fstype hybrid --s3.ak AK --s3.sk SK --s3.endpoint http://localhost:9000 --s3.bucketname test1 --s3.blocksize 4MiB --s3.chunksize 4MiB --s3.storageclass STANDARD --volume.bitmaplocation AtStart --volume.blockgroupsize 128MiB --volume.blocksize 4KiB --volume.name volume --volume.password password --volume.size 1MiB --volume.slicesize 1MiB --volume.user user`
)

type CreateFsRpc struct {
Expand Down Expand Up @@ -98,6 +98,7 @@ func (fCmd *FsCommand) AddFlags() {
config.AddS3BucknameOptionFlag(fCmd.Cmd)
config.AddS3BlocksizeOptionFlag(fCmd.Cmd)
config.AddS3ChunksizeOptionFlag(fCmd.Cmd)
config.AddS3StorageclassOptionFlag(fCmd.Cmd)
// volume
config.AddVolumeSizeOptionFlag(fCmd.Cmd)
config.AddVolumeBlockgroupsizeOptionFlag(fCmd.Cmd)
Expand Down Expand Up @@ -209,14 +210,19 @@ func setS3Info(detail *mds.FsDetail, cmd *cobra.Command) *cmderror.CmdError {
errParse.Format(config.CURVEFS_S3_CHUNKSIZE, chunksizeStr)
return errParse
}

storageclassStr := config.GetFlagString(cmd, config.CURVEFS_S3_STORAGECLASS)
storageclass, errStorageclass := cobrautil.TranslateStorageClass(storageclassStr)
if errStorageclass.TypeCode() != cmderror.CODE_SUCCESS {
return errStorageclass
}
info := &common.S3Info{
Ak: &ak,
Sk: &sk,
Endpoint: &endpoint,
Bucketname: &bucketname,
BlockSize: &blocksize,
ChunkSize: &chunksize,
Ak: &ak,
Sk: &sk,
Endpoint: &endpoint,
Bucketname: &bucketname,
BlockSize: &blocksize,
ChunkSize: &chunksize,
StorageClass: &storageclass,
}
detail.S3Info = info
return cmderror.ErrSuccess()
Expand Down
34 changes: 22 additions & 12 deletions tools-v2/pkg/config/fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,9 @@ const (
CURVEFS_S3_CHUNKSIZE = "s3.chunksize"
VIPER_CURVEFS_S3CHUNKSIZE = "curvefs.s3.chunksize"
CURVEFS_DEFAULT_S3_CHUNKSIZE = "64 MiB"
CURVEFS_S3_STORAGECLASS = "s3.storageclass"
VIPER_CURVEFS_S3_STORAGECLASS = "curvefs.s3.storageclass"
CURVEFS_DEFAULT_STORAGECLASS = "NOT_SET"
// Volume
CURVEFS_VOLUME_SIZE = "volume.size"
VIPER_CURVEFS_VOLUME_SIZE = "curvefs.volume.size"
Expand Down Expand Up @@ -174,12 +177,13 @@ var (
CURVEFS_STORAGE: VIPER_CURVEFS_STORAGE,

// S3
CURVEFS_S3_AK: VIPER_CURVEFS_S3_AK,
CURVEFS_S3_SK: VIPER_CURVEFS_S3_SK,
CURVEFS_S3_ENDPOINT: VIPER_CURVEFS_S3_ENDPOINT,
CURVEFS_S3_BUCKETNAME: VIPER_CURVEFS_S3_BUCKETNAME,
CURVEFS_S3_BLOCKSIZE: VIPER_CURVEFS_S3_BLOCKSIZE,
CURVEFS_S3_CHUNKSIZE: VIPER_CURVEFS_S3CHUNKSIZE,
CURVEFS_S3_AK: VIPER_CURVEFS_S3_AK,
CURVEFS_S3_SK: VIPER_CURVEFS_S3_SK,
CURVEFS_S3_ENDPOINT: VIPER_CURVEFS_S3_ENDPOINT,
CURVEFS_S3_BUCKETNAME: VIPER_CURVEFS_S3_BUCKETNAME,
CURVEFS_S3_BLOCKSIZE: VIPER_CURVEFS_S3_BLOCKSIZE,
CURVEFS_S3_CHUNKSIZE: VIPER_CURVEFS_S3CHUNKSIZE,
CURVEFS_S3_STORAGECLASS: VIPER_CURVEFS_S3_STORAGECLASS,

// Volume
CURVEFS_VOLUME_SIZE: VIPER_CURVEFS_VOLUME_SIZE,
Expand All @@ -204,12 +208,13 @@ var (
CURVEFS_STORAGE: CURVEFS_DEFAULT_STORAGE,

// S3
CURVEFS_S3_AK: CURVEFS_DEFAULT_S3_AK,
CURVEFS_S3_SK: CURVEFS_DEFAULT_S3_SK,
CURVEFS_S3_ENDPOINT: CURVEFS_DEFAULT_ENDPOINT,
CURVEFS_S3_BUCKETNAME: CURVEFS_DEFAULT_S3_BUCKETNAME,
CURVEFS_S3_BLOCKSIZE: CURVEFS_DEFAULT_S3_BLOCKSIZE,
CURVEFS_S3_CHUNKSIZE: CURVEFS_DEFAULT_S3_CHUNKSIZE,
CURVEFS_S3_AK: CURVEFS_DEFAULT_S3_AK,
CURVEFS_S3_SK: CURVEFS_DEFAULT_S3_SK,
CURVEFS_S3_ENDPOINT: CURVEFS_DEFAULT_ENDPOINT,
CURVEFS_S3_BUCKETNAME: CURVEFS_DEFAULT_S3_BUCKETNAME,
CURVEFS_S3_BLOCKSIZE: CURVEFS_DEFAULT_S3_BLOCKSIZE,
CURVEFS_S3_CHUNKSIZE: CURVEFS_DEFAULT_S3_CHUNKSIZE,
CURVEFS_S3_STORAGECLASS: CURVEFS_DEFAULT_STORAGECLASS,

// Volume
CURVEFS_VOLUME_SIZE: CURVEFS_DEFAULT_VOLUME_SIZE,
Expand Down Expand Up @@ -676,6 +681,11 @@ func AddS3ChunksizeOptionFlag(cmd *cobra.Command) {
AddStringOptionFlag(cmd, CURVEFS_S3_CHUNKSIZE, "s3 chunksize")
}

// S3.StorageClass [option]
func AddS3StorageclassOptionFlag(cmd *cobra.Command) {
AddStringOptionFlag(cmd, CURVEFS_S3_STORAGECLASS, "s3 storageclass")
}

// volume.size [option]
func AddVolumeSizeOptionFlag(cmd *cobra.Command) {
AddStringOptionFlag(cmd, CURVEFS_VOLUME_SIZE, "volume size")
Expand Down

0 comments on commit e0eb79f

Please sign in to comment.