feature: support for Netease Cloud NOS storage backend (#12)

Gopkg.toml

@@ -69,6 +69,10 @@
   branch = "master"
   name = "github.com/tencentyun/cos-go-sdk-v5"
 
+[[constraint]]
+  branch = "master"
+  name = "github.com/NetEase-Object-Storage/nos-golang-sdk"
+
 [prune]
   go-tests = true
   unused-packages = true

netease.go

@@ -0,0 +1,209 @@
+/*
+Copyright The Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	pathutil "path"
+
+	"time"
+
+	"github.com/NetEase-Object-Storage/nos-golang-sdk/config"
+	"github.com/NetEase-Object-Storage/nos-golang-sdk/logger"
+	"github.com/NetEase-Object-Storage/nos-golang-sdk/model"
+	"github.com/NetEase-Object-Storage/nos-golang-sdk/nosclient"
+)
+
+// NeteaseNOSBackend is a storage backend for Netease Cloud NOS
+type NeteaseNOSBackend struct {
+	Client nosclient.NosClient
+	Bucket string
+	Prefix string
+}
+
+// NewNeteaseNOSBackend creates a new instance of NeteaseNOSBackend
+func NewNeteaseNOSBackend(bucket string, prefix string, endpoint string) *NeteaseNOSBackend {
+	accessKeyId := os.Getenv("NETEASE_CLOUD_ACCESS_KEY_ID")
+	accessKeySecret := os.Getenv("NETEASE_CLOUD_ACCESS_KEY_SECRET")
+
+	if len(accessKeyId) == 0 {
+		panic("NETEASE_CLOUD_ACCESS_KEY_ID environment variable is not set")
+	}
+
+	if len(accessKeySecret) == 0 {
+		panic("NETEASE_CLOUD_ACCESS_KEY_SECRET environment variable is not set")
+	}
+
+	if len(endpoint) == 0 {
+		// Set default endpoint
+		endpoint = "nos-eastchina1.126.net"
+	}
+
+	conf := &config.Config{
+		Endpoint:                    endpoint,
+		AccessKey:                   accessKeyId,
+		SecretKey:                   accessKeySecret,
+		NosServiceConnectTimeout:    3,
+		NosServiceReadWriteTimeout:  5,
+		NosServiceMaxIdleConnection: 15,
+		LogLevel:                    logger.LogLevel(logger.DEBUG),
+		Logger:                      logger.NewDefaultLogger(),
+	}
+
+	client, err := nosclient.New(conf)
+	if err != nil {
+		panic("Failed to create NOS client: " + err.Error())
+	}
+
+	b := &NeteaseNOSBackend{
+		Client: *client,
+		Bucket: bucket,
+		Prefix: prefix,
+	}
+	return b
+}
+
+// ListObjects lists all objects in Netease Cloud NOS bucket, at prefix
+func (b NeteaseNOSBackend) ListObjects(prefix string) ([]Object, error) {
+	var objects []Object
+
+	prefix = pathutil.Join(b.Prefix, prefix)
+
+	listRequest := &model.ListObjectsRequest{
+		Bucket:    b.Bucket,
+		Prefix:    prefix,
+		Delimiter: "",
+		Marker:    "",
+		MaxKeys:   100,
+	}
+
+	for {
+		var lor *model.ListObjectsResult
+		lor, err := b.Client.ListObjects(listRequest)
+		if err != nil {
+			return objects, nil
+		}
+
+		for _, obj := range lor.Contents {
+			path := removePrefixFromObjectPath(prefix, obj.Key)
+			if objectPathIsInvalid(path) {
+				continue
+			}
+
+			local, _ := time.LoadLocation("Local")
+			// LastModified time layout in NOS is 2006-01-02T15:04:05 -0700
+			t, _ := time.ParseInLocation("2006-01-02T15:04:05 -0700", obj.LastModified, local)
+			object := Object{
+				Path:         path,
+				Content:      []byte{},
+				LastModified: t,
+			}
+			objects = append(objects, object)
+		}
+		if !lor.IsTruncated {
+			break
+		}
+
+	}
+
+	return objects, nil
+}
+
+// GetObject retrieves an object from Netease Cloud NOS bucket, at prefix
+func (b NeteaseNOSBackend) GetObject(path string) (Object, error) {
+	var object Object
+	object.Path = path
+	var content []byte
+	key := pathutil.Join(b.Prefix, path)
+
+	objectRequest := &model.GetObjectRequest{
+		Bucket: b.Bucket,
+		Object: key,
+	}
+
+	var nosObject *model.NOSObject
+	nosObject, err := b.Client.GetObject(objectRequest)
+	if err != nil {
+		return object, err
+	}
+
+	body := nosObject.Body
+	content, err = ioutil.ReadAll(body)
+	defer body.Close()
+	if err != nil {
+		return object, err
+	}
+
+	object.Content = content
+	objectMetaRequest := &model.ObjectRequest{
+		Bucket: b.Bucket,
+		Object: key,
+	}
+
+	var meta *model.ObjectMetadata
+	meta, err = b.Client.GetObjectMetaData(objectMetaRequest)
+	if err != nil {
+		return object, err
+	}
+
+	m := meta.Metadata
+	// 	"Last-Modified" 是从nos获取的存储 最后修改时间 的key
+	if t, ok := m["Last-Modified"]; ok {
+
+		local, _ := time.LoadLocation("Local")
+		// NOS的LastModified格式为 2019-04-18T16:55:39 +0800
+		lastModified, _ := time.ParseInLocation("2006-01-02T15:04:05 -0700", t, local)
+		object.LastModified = lastModified
+	}
+
+	return object, nil
+}
+
+// PutObject uploads an object to Netease Cloud NOS bucket, at prefix
+func (b NeteaseNOSBackend) PutObject(path string, content []byte) error {
+	key := pathutil.Join(b.Prefix, path)
+	var err error
+
+	metadata := &model.ObjectMetadata{
+		Metadata:      map[string]string{},
+		ContentLength: int64(len(content)),
+	}
+
+	putObjectRequest := &model.PutObjectRequest{
+		Bucket:   b.Bucket,
+		Object:   key,
+		Body:     bytes.NewReader(content),
+		Metadata: metadata,
+	}
+	_, err = b.Client.PutObjectByStream(putObjectRequest)
+	return err
+}
+
+// DeleteObject removes an object from Netease Cloud NOS bucket, at prefix
+func (b NeteaseNOSBackend) DeleteObject(path string) error {
+	key := pathutil.Join(b.Prefix, path)
+
+	objectRequest := &model.ObjectRequest{
+		Bucket: b.Bucket,
+		Object: key,
+	}
+
+	err := b.Client.DeleteObject(objectRequest)
+	return err
+}

netease_test.go

@@ -0,0 +1,89 @@
+/*
+Copyright The Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"os"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type NeteaseTestSuite struct {
+	suite.Suite
+	BrokenNeteaseNOSBackend   *NeteaseNOSBackend
+	NoPrefixNeteaseNOSBackend *NeteaseNOSBackend
+}
+
+const nosTestCount = 100
+
+func (suite *NeteaseTestSuite) SetupSuite() {
+	backend := NewNeteaseNOSBackend("fake-container-cant-exist-fbce123", "", "")
+	suite.BrokenNeteaseNOSBackend = backend
+
+	nosBucket := os.Getenv("TEST_STORAGE_NETEASE_BUCKET")
+	nosEndpoint := os.Getenv("TEST_STORAGE_NETEASE_ENDPOINT")
+	backend = NewNeteaseNOSBackend(nosBucket, "", nosEndpoint)
+	suite.NoPrefixNeteaseNOSBackend = backend
+
+	data := []byte("some object")
+	path := "deleteme.txt"
+
+	for i := 0; i < nosTestCount; i++ {
+		newPath := strconv.Itoa(i) + path
+		err := suite.NoPrefixNeteaseNOSBackend.PutObject(newPath, data)
+		suite.Nil(err, "no error putting deleteme.txt using Netease Cloud NOS backend")
+	}
+}
+
+func (suite *NeteaseTestSuite) TearDownSuite() {
+	path := "deleteme.txt"
+	for i := 0; i < bosTestCount; i++ {
+		newPath := strconv.Itoa(i) + path
+
+		err := suite.NoPrefixNeteaseNOSBackend.DeleteObject(newPath)
+		suite.Nil(err, "no error deleting deleteme.txt using Netease NOS backend")
+	}
+}
+
+func (suite *NeteaseTestSuite) TestListObjects() {
+	_, err := suite.BrokenNeteaseNOSBackend.ListObjects("")
+	suite.NotNil(err, "cannot list objects with bad bucket")
+
+	objs, err := suite.NoPrefixNeteaseNOSBackend.ListObjects("")
+	suite.Nil(err, "can list objects with good bucket, no prefix")
+	suite.Equal(len(objs), nosTestCount, "able to list objects")
+}
+
+func (suite *NeteaseTestSuite) TestGetObject() {
+	_, err := suite.BrokenNeteaseNOSBackend.GetObject("this-file-cannot-possibly-exist.tgz")
+	suite.NotNil(err, "cannot get objects with bad bucket")
+}
+
+func (suite *NeteaseTestSuite) TestPutObject() {
+	err := suite.BrokenNeteaseNOSBackend.PutObject("this-file-will-not-upload.txt", []byte{})
+	suite.NotNil(err, "cannot put objects with bad bucket")
+}
+
+func TestNeteaseStorageTestSuite(t *testing.T) {
+	if os.Getenv("TEST_CLOUD_STORAGE") == "1" &&
+		os.Getenv("TEST_STORAGE_NETEASE_BUCKET") != "" &&
+		os.Getenv("TEST_STORAGE_NETEASE_ENDPOINT") != "" {
+		suite.Run(t, new(NeteaseTestSuite))
+	}
+}

storage_test.go

@@ -58,6 +58,8 @@ func (suite *StorageTestSuite) setupStorageBackends() {
 		bosEndpoint := os.Getenv("TEST_STORAGE_BAIDU_ENDPOINT")
 		cosBucket := os.Getenv("TEST_STORAGE_TENCENT_BUCKET")
 		cosEndpoint := os.Getenv("TEST_STORAGE_TENCENT_ENDPOINT")
+		nosBucket := os.Getenv("TEST_STORAGE_NETEASE_BUCKET")
+		nosEndpoint := os.Getenv("TEST_STORAGE_NETEASE_ENDPOINT")
 		if s3Bucket != "" && s3Region != "" {
 			suite.StorageBackends["AmazonS3"] = Backend(NewAmazonS3Backend(s3Bucket, prefix, s3Region, "", ""))
 		}
@@ -82,6 +84,9 @@ func (suite *StorageTestSuite) setupStorageBackends() {
 		if cosBucket != "" {
 			suite.StorageBackends["TencentCloudCOS"] = Backend(NewTencentCloudCOSBackend(cosBucket, prefix, cosEndpoint))
 		}
+		if nosBucket != "" {
+			suite.StorageBackends["NeteaseCloudNOS"] = Backend(NewNeteaseNOSBackend(nosBucket, prefix, nosEndpoint))
+		}
 	}
 }