diff --git a/go.mod b/go.mod index f304ff42..e8dbe496 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.17 require ( cloud.google.com/go/storage v1.22.1 github.com/Azure/azure-sdk-for-go v65.0.0+incompatible - github.com/NetEase-Object-Storage/nos-golang-sdk v0.0.0-20191125093154-335c2b73bf6b github.com/aliyun/aliyun-oss-go-sdk v2.2.4+incompatible github.com/aws/aws-sdk-go v1.44.27 github.com/baidubce/bce-sdk-go v0.9.123 diff --git a/go.sum b/go.sum index 263d0250..3b86340f 100644 --- a/go.sum +++ b/go.sum @@ -79,8 +79,6 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NetEase-Object-Storage/nos-golang-sdk v0.0.0-20191125093154-335c2b73bf6b h1:jZIv0SILYL0wRLzAB8Ha19VDm7JnYn8t9NYbz2s2AVc= -github.com/NetEase-Object-Storage/nos-golang-sdk v0.0.0-20191125093154-335c2b73bf6b/go.mod h1:0N5CbwYI/8V1T6YOEwkgMvLmiGDNn661vLutBZQrC2c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/netease.go b/netease.go deleted file mode 100644 index e1444c2f..00000000 --- a/netease.go +++ /dev/null @@ -1,219 +0,0 @@ -/* -Copyright The Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package storage - -import ( - "bytes" - "io/ioutil" - "os" - pathutil "path" - - "time" - - "github.com/NetEase-Object-Storage/nos-golang-sdk/config" - "github.com/NetEase-Object-Storage/nos-golang-sdk/logger" - "github.com/NetEase-Object-Storage/nos-golang-sdk/model" - "github.com/NetEase-Object-Storage/nos-golang-sdk/nosclient" -) - -const ( - RFC1123_NOS = "Mon, 02 Jan 2006 15:04:05 Asia/Shanghai" - RFC1123_NOS_GMT = "Mon, 02 Jan 2006 15:04:05 GMT" -) - -// NeteaseNOSBackend is a storage backend for Netease Cloud NOS -type NeteaseNOSBackend struct { - Client nosclient.NosClient - Bucket string - Prefix string -} - -// NewNeteaseNOSBackend creates a new instance of NeteaseNOSBackend -func NewNeteaseNOSBackend(bucket string, prefix string, endpoint string) *NeteaseNOSBackend { - accessKeyId := os.Getenv("NETEASE_CLOUD_ACCESS_KEY_ID") - accessKeySecret := os.Getenv("NETEASE_CLOUD_ACCESS_KEY_SECRET") - - if len(accessKeyId) == 0 { - panic("NETEASE_CLOUD_ACCESS_KEY_ID environment variable is not set") - } - - if len(accessKeySecret) == 0 { - panic("NETEASE_CLOUD_ACCESS_KEY_SECRET environment variable is not set") - } - - if len(endpoint) == 0 { - // Set default endpoint - endpoint = "nos-eastchina1.126.net" - } - - conf := &config.Config{ - Endpoint: endpoint, - AccessKey: accessKeyId, - SecretKey: accessKeySecret, - NosServiceConnectTimeout: 3, - NosServiceReadWriteTimeout: 5, - NosServiceMaxIdleConnection: 15, - LogLevel: logger.LogLevel(logger.DEBUG), - Logger: logger.NewDefaultLogger(), - } - - client, err := nosclient.New(conf) - if err != nil { - panic("Failed to create NOS client: " + err.Error()) - } - - b := &NeteaseNOSBackend{ - Client: *client, - Bucket: bucket, - Prefix: prefix, - } - return b -} - -// ListObjects lists all objects in Netease Cloud NOS bucket, at prefix -func (b NeteaseNOSBackend) ListObjects(prefix string) ([]Object, error) { - var objects []Object - - prefix = pathutil.Join(b.Prefix, prefix) - - listRequest := &model.ListObjectsRequest{ - Bucket: b.Bucket, - Prefix: prefix, - Delimiter: "", - Marker: "", - MaxKeys: 100, - } - - for { - var lor *model.ListObjectsResult - lor, err := b.Client.ListObjects(listRequest) - if err != nil { - return objects, err - } - - for _, obj := range lor.Contents { - path := removePrefixFromObjectPath(prefix, obj.Key) - if objectPathIsInvalid(path) { - continue - } - - local, _ := time.LoadLocation("Local") - // LastModified time layout in NOS is 2006-01-02T15:04:05 -0700 - t, _ := time.ParseInLocation("2006-01-02T15:04:05 -0700", obj.LastModified, local) - object := Object{ - Path: path, - Content: []byte{}, - LastModified: t, - } - objects = append(objects, object) - } - if !lor.IsTruncated { - break - } - - } - - return objects, nil -} - -// GetObject retrieves an object from Netease Cloud NOS bucket, at prefix -func (b NeteaseNOSBackend) GetObject(path string) (Object, error) { - var object Object - object.Path = path - var content []byte - key := pathutil.Join(b.Prefix, path) - - objectRequest := &model.GetObjectRequest{ - Bucket: b.Bucket, - Object: key, - } - - var nosObject *model.NOSObject - nosObject, err := b.Client.GetObject(objectRequest) - if err != nil { - return object, err - } - - body := nosObject.Body - content, err = ioutil.ReadAll(body) - defer body.Close() - if err != nil { - return object, err - } - - object.Content = content - objectMetaRequest := &model.ObjectRequest{ - Bucket: b.Bucket, - Object: key, - } - - var meta *model.ObjectMetadata - meta, err = b.Client.GetObjectMetaData(objectMetaRequest) - if err != nil { - return object, err - } - - m := meta.Metadata - // "Last-Modified" is the key for last modified time。format is "Thu, 18 Jun 2020 10:53:53 GMT" - if t, ok := m["Last-Modified"]; ok { - modTime, err := time.Parse(RFC1123_NOS_GMT, t) - if err != nil { - // RFC1123 is CST time, should reduce 8 hours - modTime, err = time.Parse(RFC1123_NOS, t) - if err != nil { - return object, err - } - modTime.Add(-8 * time.Hour) - } - object.LastModified = modTime - } - - return object, nil -} - -// PutObject uploads an object to Netease Cloud NOS bucket, at prefix -func (b NeteaseNOSBackend) PutObject(path string, content []byte) error { - key := pathutil.Join(b.Prefix, path) - var err error - - metadata := &model.ObjectMetadata{ - Metadata: map[string]string{}, - ContentLength: int64(len(content)), - } - - putObjectRequest := &model.PutObjectRequest{ - Bucket: b.Bucket, - Object: key, - Body: bytes.NewReader(content), - Metadata: metadata, - } - _, err = b.Client.PutObjectByStream(putObjectRequest) - return err -} - -// DeleteObject removes an object from Netease Cloud NOS bucket, at prefix -func (b NeteaseNOSBackend) DeleteObject(path string) error { - key := pathutil.Join(b.Prefix, path) - - objectRequest := &model.ObjectRequest{ - Bucket: b.Bucket, - Object: key, - } - - err := b.Client.DeleteObject(objectRequest) - return err -} diff --git a/netease_test.go b/netease_test.go deleted file mode 100644 index df5db4f8..00000000 --- a/netease_test.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright The Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package storage - -import ( - "os" - "strconv" - "testing" - - "github.com/stretchr/testify/suite" -) - -type NeteaseTestSuite struct { - suite.Suite - BrokenNeteaseNOSBackend *NeteaseNOSBackend - NoPrefixNeteaseNOSBackend *NeteaseNOSBackend -} - -const nosTestCount = 100 - -func (suite *NeteaseTestSuite) SetupSuite() { - backend := NewNeteaseNOSBackend("fake-container-cant-exist-fbce123", "", "") - suite.BrokenNeteaseNOSBackend = backend - - nosBucket := os.Getenv("TEST_STORAGE_NETEASE_BUCKET") - nosEndpoint := os.Getenv("TEST_STORAGE_NETEASE_ENDPOINT") - backend = NewNeteaseNOSBackend(nosBucket, "", nosEndpoint) - suite.NoPrefixNeteaseNOSBackend = backend - - data := []byte("some object") - path := "deleteme.txt" - - for i := 0; i < nosTestCount; i++ { - newPath := strconv.Itoa(i) + path - err := suite.NoPrefixNeteaseNOSBackend.PutObject(newPath, data) - suite.Nil(err, "no error putting deleteme.txt using Netease Cloud NOS backend") - } -} - -func (suite *NeteaseTestSuite) TearDownSuite() { - path := "deleteme.txt" - for i := 0; i < bosTestCount; i++ { - newPath := strconv.Itoa(i) + path - - err := suite.NoPrefixNeteaseNOSBackend.DeleteObject(newPath) - suite.Nil(err, "no error deleting deleteme.txt using Netease NOS backend") - } -} - -func (suite *NeteaseTestSuite) TestListObjects() { - _, err := suite.BrokenNeteaseNOSBackend.ListObjects("") - suite.NotNil(err, "cannot list objects with bad bucket") - - objs, err := suite.NoPrefixNeteaseNOSBackend.ListObjects("") - suite.Nil(err, "can list objects with good bucket, no prefix") - suite.Equal(len(objs), nosTestCount, "able to list objects") -} - -func (suite *NeteaseTestSuite) TestGetObject() { - _, err := suite.BrokenNeteaseNOSBackend.GetObject("this-file-cannot-possibly-exist.tgz") - suite.NotNil(err, "cannot get objects with bad bucket") -} - -func (suite *NeteaseTestSuite) TestPutObject() { - err := suite.BrokenNeteaseNOSBackend.PutObject("this-file-will-not-upload.txt", []byte{}) - suite.NotNil(err, "cannot put objects with bad bucket") -} - -func TestNeteaseStorageTestSuite(t *testing.T) { - if os.Getenv("TEST_CLOUD_STORAGE") == "1" && - os.Getenv("TEST_STORAGE_NETEASE_BUCKET") != "" && - os.Getenv("TEST_STORAGE_NETEASE_ENDPOINT") != "" { - suite.Run(t, new(NeteaseTestSuite)) - } -} diff --git a/storage_test.go b/storage_test.go index 0391b559..79cce289 100644 --- a/storage_test.go +++ b/storage_test.go @@ -38,7 +38,7 @@ func (suite *StorageTestSuite) setupStorageBackends() { suite.StorageBackends["LocalFilesystem"] = Backend(NewLocalFilesystemBackend(suite.TempDirectory)) // create empty dir in local storage to make sure it doesnt end up in ListObjects - err := os.MkdirAll(fmt.Sprintf("%s/%s", suite.TempDirectory, "ignoreme"), 0777) + err := os.MkdirAll(fmt.Sprintf("%s/%s", suite.TempDirectory, "ignoreme"), 0o777) suite.Nil(err, "No error creating ignored dir in local storage") if os.Getenv("TEST_CLOUD_STORAGE") == "1" { @@ -58,8 +58,6 @@ func (suite *StorageTestSuite) setupStorageBackends() { bosEndpoint := os.Getenv("TEST_STORAGE_BAIDU_ENDPOINT") cosBucket := os.Getenv("TEST_STORAGE_TENCENT_BUCKET") cosEndpoint := os.Getenv("TEST_STORAGE_TENCENT_ENDPOINT") - nosBucket := os.Getenv("TEST_STORAGE_NETEASE_BUCKET") - nosEndpoint := os.Getenv("TEST_STORAGE_NETEASE_ENDPOINT") if s3Bucket != "" && s3Region != "" { suite.StorageBackends["AmazonS3"] = Backend(NewAmazonS3Backend(s3Bucket, prefix, s3Region, "", "")) } @@ -84,9 +82,6 @@ func (suite *StorageTestSuite) setupStorageBackends() { if cosBucket != "" { suite.StorageBackends["TencentCloudCOS"] = Backend(NewTencentCloudCOSBackend(cosBucket, prefix, cosEndpoint)) } - if nosBucket != "" { - suite.StorageBackends["NeteaseCloudNOS"] = Backend(NewNeteaseNOSBackend(nosBucket, prefix, nosEndpoint)) - } } } @@ -238,7 +233,6 @@ func (suite *StorageTestSuite) TestGetObjectSliceDiff() { suite.Empty(diff.Removed, "removed slice empty") suite.Equal(diff.Added, []Object{os2[1]}, "added slice empty") suite.Empty(diff.Updated, "updated slice empty") - } func TestStorageTestSuite(t *testing.T) {