From e332362ec59724e143725dc5a5a0dacae3be73be Mon Sep 17 00:00:00 2001 From: will-2012 <117156346+will-2012@users.noreply.github.com> Date: Mon, 13 Mar 2023 11:49:46 +0800 Subject: [PATCH] feat: add check billing (#173) * feat: add check billing * fix: update by comment tips * fix: fix ec piece key error * fix: polish by comment tips * fix: update comments content --------- Co-authored-by: joeylichang --- config/subconfig.go | 1 - deployment/localup/localup.sh | 11 +- go.mod | 29 ++-- go.sum | 48 +++---- model/const.go | 51 ++++--- model/errors/rpc_error.go | 23 +++- pkg/greenfield/auth_service.go | 127 ------------------ pkg/greenfield/query_service.go | 35 ++++- pkg/stream/payload_stream.go | 2 +- proto/service/challenge/types/challenge.proto | 6 +- .../service/downloader/types/downloader.proto | 34 ++--- .../service/syncer/types/syncer_service.proto | 8 +- .../uploader/types/uploader_service.proto | 24 ++-- service/challenge/challenge_service.go | 4 +- service/challenge/client/challenge_client.go | 8 +- .../downloader/client/downloader_client.go | 8 +- service/downloader/downloader.go | 8 -- service/downloader/downloader_config.go | 2 - service/downloader/downloader_service.go | 44 +++--- service/gateway/admin_handler.go | 21 ++- service/gateway/client/gateway_client.go | 4 +- service/gateway/gateway.go | 3 +- service/gateway/gateway_config.go | 2 +- service/gateway/object_handler.go | 52 +++---- service/gateway/request_util.go | 81 ++++++----- service/gateway/response_util.go | 41 ++++-- service/gateway/sync_piece_handler.go | 16 +-- service/syncer/syncer_service.go | 4 +- service/uploader/client/uploader_client.go | 14 +- service/uploader/uploader_service.go | 16 +-- store/piecestore/piece/piece_store.go | 2 +- store/piecestore/storage/disk_file.go | 2 +- store/piecestore/storage/memory.go | 2 +- store/piecestore/storage/memory_test.go | 2 +- store/piecestore/storage/s3.go | 2 +- store/sqldb/entity.go | 2 +- store/sqldb/sp.go | 42 ++++-- store/sqldb/sp_schema.go | 2 +- store/sqldb/traffic.go | 29 ++-- store/sqldb/traffic_schema.go | 4 +- 40 files changed, 396 insertions(+), 420 deletions(-) delete mode 100644 pkg/greenfield/auth_service.go diff --git a/config/subconfig.go b/config/subconfig.go index bb8a352d7..ef0709ad7 100644 --- a/config/subconfig.go +++ b/config/subconfig.go @@ -88,7 +88,6 @@ func (cfg *StorageProviderConfig) MakeUploaderConfig() (*uploader.UploaderConfig func (cfg *StorageProviderConfig) MakeDownloaderConfig() (*downloader.DownloaderConfig, error) { dCfg := &downloader.DownloaderConfig{ SpDBConfig: cfg.SpDBConfig, - ChainConfig: cfg.ChainConfig, PieceStoreConfig: cfg.PieceStoreConfig, } if _, ok := cfg.GRPCAddress[model.DownloaderService]; ok { diff --git a/deployment/localup/localup.sh b/deployment/localup/localup.sh index 2010e5acb..486446109 100644 --- a/deployment/localup/localup.sh +++ b/deployment/localup/localup.sh @@ -81,7 +81,7 @@ make_config() { sed -i -e "s/9533/$(($cur_port+533))/g" config.toml sed -i -e "s/9633/$(($cur_port+633))/g" config.toml sed -i -e "s/9733/$(($cur_port+733))/g" config.toml - sed -i -e "s/SP_OPERATOR_PUB_KEY/${OPERATOR_ADDRESS}/g" config.toml + sed -i -e "s/SpOperatorAddress = \".*\"/SpOperatorAddress = \"${OPERATOR_ADDRESS}\"/g" config.toml sed -i -e "s/OperatorPrivateKey = \".*\"/OperatorPrivateKey = \"${OPERATOR_PRIVATE_KEY}\"/g" config.toml sed -i -e "s/FundingPrivateKey = \".*\"/FundingPrivateKey = \"${FUNDING_PRIVATE_KEY}\"/g" config.toml sed -i -e "s/SealPrivateKey = \".*\"/SealPrivateKey = \"${SEAL_PRIVATE_KEY}\"/g" config.toml @@ -91,7 +91,7 @@ make_config() { sed -i -e "s/localhost\:9090/${CHAIN_GRPC_ENDPOINT}/g" config.toml sed -i -e "s/localhost\:26750/${CHAIN_HTTP_ENDPOINT}/g" config.toml echo "succeed to generate config.toml in "${sp_dir} - cd - + cd - >/dev/null index=$(($index+1)) done } @@ -104,7 +104,8 @@ start_sp() { for sp_dir in ${workspace}/${SP_DEPLOY_DIR}/* ; do cd ${sp_dir} nohup ./${sp_bin_name}${index} --config config.toml log.txt 2>&1& - cd - + echo "succeed to start sp in "${sp_dir} + cd - >/dev/null index=$(($index+1)) done echo "succeed to start storage providers" @@ -130,7 +131,7 @@ reset_db() { mysql -u ${USER} -h ${hostname} -P ${port} -p${PWD} -e "drop database if exists ${DATABASE}" mysql -u ${USER} -h ${hostname} -P ${port} -p${PWD} -e "create database ${DATABASE}" echo "succeed to reset db in "${sp_dir} - cd - + cd - >/dev/null done } @@ -142,7 +143,7 @@ reset_store() { cd ${sp_dir} rm -rf ./data echo "succeed to reset store in "${sp_dir} - cd - + cd - >/dev/null done } diff --git a/go.mod b/go.mod index 4d5c71903..10e8be6c9 100644 --- a/go.mod +++ b/go.mod @@ -3,28 +3,27 @@ module github.com/bnb-chain/greenfield-storage-provider go 1.19 replace ( - // TODO: point to develop branch, will be changed to v0.0.6 after greenfield v0.0.6 released - github.com/bnb-chain/greenfield => github.com/bnb-chain/greenfield v0.0.0-20230220084048-b09d0197ee41 - // TODO: point to develop branch, will be changed to v0.0.6 after greenfield-sdk-go v0.0.6 released + cosmossdk.io/math => github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20230228075616-68ac309b432c + github.com/bnb-chain/greenfield => github.com/bnb-chain/greenfield v0.0.7 github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0 - // TODO: point to develop branch, will be changed to v0.0.7 after greenfield-sdk-go v0.0.7 released - github.com/cosmos/cosmos-sdk => github.com/bnb-chain/gnfd-cosmos-sdk v0.0.2-0.20230216105555-b94a14479b11 + //github.com/cosmos/cosmos-sdk => github.com/bnb-chain/gnfd-cosmos-sdk v0.0.9 + github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v0.0.9 github.com/forbole/juno/v4 => github.com/bnb-chain/juno/v4 v4.0.0-20230223090130-98858670d635 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 - github.com/tendermint/tendermint => github.com/bnb-chain/gnfd-tendermint v0.0.1 + github.com/tendermint/tendermint => github.com/bnb-chain/gnfd-tendermint v0.0.2 ) require ( cosmossdk.io/math v1.0.0-beta.6 github.com/aws/aws-sdk-go v1.44.159 - github.com/bnb-chain/greenfield v0.0.5 + github.com/bnb-chain/greenfield v0.0.6 github.com/bnb-chain/greenfield-common/go v0.0.0-20230227091017-b60c7936a1c6 github.com/bnb-chain/greenfield-go-sdk v0.0.5 github.com/bytedance/gopkg v0.0.0-20221122125632-68358b8ecec6 github.com/cloudflare/cfssl v1.6.3 github.com/cosmos/cosmos-proto v1.0.0-beta.1 github.com/cosmos/cosmos-sdk v0.46.7 - github.com/cosmos/gogoproto v1.4.4 + github.com/cosmos/gogoproto v1.4.6 github.com/ethereum/go-ethereum v1.10.19 github.com/forbole/juno/v4 v4.0.0-00010101000000-000000000000 github.com/gin-gonic/gin v1.8.2 @@ -42,8 +41,8 @@ require ( github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 go.uber.org/multierr v1.9.0 go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20230111222715-75897c7a292a - google.golang.org/grpc v1.52.3 + golang.org/x/exp v0.0.0-20230131160201-f062dba9d201 + google.golang.org/grpc v1.53.0 gorm.io/driver/mysql v1.4.6 gorm.io/gorm v1.24.5 ) @@ -63,7 +62,7 @@ require ( github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/cockroachdb/apd/v2 v2.0.2 // indirect github.com/confio/ics23/go v0.9.0 // indirect @@ -99,9 +98,10 @@ require ( github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gogo/gateway v1.1.0 // indirect github.com/golang/glog v1.0.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/orderedcode v0.0.1 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -188,13 +188,12 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.10.0 // indirect golang.org/x/crypto v0.4.0 // indirect - golang.org/x/net v0.6.0 // indirect + golang.org/x/net v0.7.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect - golang.org/x/tools v0.6.0 // indirect - google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 // indirect + google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 // indirect google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 7f5583df3..bc79963ed 100644 --- a/go.sum +++ b/go.sum @@ -61,8 +61,6 @@ contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0Wk contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= cosmossdk.io/errors v1.0.0-beta.7 h1:gypHW76pTQGVnHKo6QBkb4yFOJjC+sUGRc5Al3Odj1w= cosmossdk.io/errors v1.0.0-beta.7/go.mod h1:mz6FQMJRku4bY7aqS/Gwfcmr/ue91roMEKAmDUDpBfE= -cosmossdk.io/math v1.0.0-beta.6 h1:WF29SiFYNde5eYvqO2kdOM9nYbDb44j3YW5B8M1m9KE= -cosmossdk.io/math v1.0.0-beta.6/go.mod h1:gUVtWwIzfSXqcOT+lBVz2jyjfua8DoBdzRsIyaUAT/8= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= @@ -204,14 +202,16 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= -github.com/bnb-chain/gnfd-cosmos-sdk v0.0.2-0.20230216105555-b94a14479b11 h1:2BDa9ntopNBk6kQWoGTbX6TzJckyek0K9bk9NcHBIhw= -github.com/bnb-chain/gnfd-cosmos-sdk v0.0.2-0.20230216105555-b94a14479b11/go.mod h1:NlIOmju3uhTOJ2YAzLPidpmh7sAgJ+J9dkUlSysHmjw= -github.com/bnb-chain/gnfd-tendermint v0.0.1 h1:KWFuxWv8m6t1vUi3ADyVcEG0qVnEzQt1tjA6dk/b7+Q= -github.com/bnb-chain/gnfd-tendermint v0.0.1/go.mod h1:/v9z9F6cq0+f7EGG92lYSLBcPYQDILoK91X8YM28hWo= -github.com/bnb-chain/greenfield v0.0.0-20230220084048-b09d0197ee41 h1:DD/1yLWJZ19xi3ofWy637sCD7ie7Ef/0QQLm1sy5tkw= -github.com/bnb-chain/greenfield v0.0.0-20230220084048-b09d0197ee41/go.mod h1:+7l0FQSh1UJyBlByIsgLsrMxrdqycCVY1xwLbZIJCno= +github.com/bnb-chain/gnfd-tendermint v0.0.2 h1:2Oy7B5A7W/55teZNhlrFR9SamYNjCXAQNjwsbt534jM= +github.com/bnb-chain/gnfd-tendermint v0.0.2/go.mod h1:/v9z9F6cq0+f7EGG92lYSLBcPYQDILoK91X8YM28hWo= +github.com/bnb-chain/greenfield v0.0.7 h1:5dKYeyJKDaigRsmaf2VT3jS2dORoafz0O2+pAEp6AdU= +github.com/bnb-chain/greenfield v0.0.7/go.mod h1:pv7p36TMwTQRG0oZx1YaDfEsN1gTiBiTaX0FQLEnBxk= github.com/bnb-chain/greenfield-common/go v0.0.0-20230227091017-b60c7936a1c6 h1:YFJz8O8p36qbNqXfzQ16Fh08U/N70qbRGNFNw7hNZT4= github.com/bnb-chain/greenfield-common/go v0.0.0-20230227091017-b60c7936a1c6/go.mod h1:Nzpqn+BK8P1Ub3Tgn300bHmWMUk9R6cBwvmasVY25J8= +github.com/bnb-chain/greenfield-cosmos-sdk v0.0.9 h1:iPMWJjkOD96jU1lJNx7XHym961s+eY4UNbS3pdpR8yo= +github.com/bnb-chain/greenfield-cosmos-sdk v0.0.9/go.mod h1:C0poOez5FBpH4Y/oa9jmco2fRojeMsJZGOZEX+GOv8w= +github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20230228075616-68ac309b432c h1:BLmdYaj7Dx0YOhfk77+KPPJSMCwpQl6f4Y30+801bf0= +github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20230228075616-68ac309b432c/go.mod h1:u/MXvf8wbUbCsAEyQSSYXXMsczAsFX48e2D6JI86T4o= github.com/bnb-chain/greenfield-go-sdk v0.0.5 h1:DpAyBi3o6n8dEtGkh31Yl+yeH6Gv2lE+qbuEsliz6xE= github.com/bnb-chain/greenfield-go-sdk v0.0.5/go.mod h1:JxsUzwHwvGMQbqRdswr+r5anLyPLb2VcyB8SqncydlU= github.com/bnb-chain/juno/v4 v4.0.0-20230223090130-98858670d635 h1:jyOjg0KlLtKWN+ah1XXeRQMVDK4NzVx8jktz/b/sK+Y= @@ -264,8 +264,9 @@ github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -322,8 +323,8 @@ github.com/cosmos/cosmos-sdk/ics23/go v0.8.0/go.mod h1:2a4dBq88TUoqoWAU5eu0lGvpF github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= -github.com/cosmos/gogoproto v1.4.4 h1:nVAsgLlAf5jeN0fV7hRlkZvf768zU+dy4pG+hxc2P34= -github.com/cosmos/gogoproto v1.4.4/go.mod h1:/yl6/nLwsZcZ2JY3OrqjRqvqCG9InUMcXRfRjQiF9DU= +github.com/cosmos/gogoproto v1.4.6 h1:Ee7z15dWJaGlgM2rWrK8N2IX7PQcuccu8oG68jp5RL4= +github.com/cosmos/gogoproto v1.4.6/go.mod h1:VS/ASYmPgv6zkPKLjR9EB91lwbLHOzaGCirmKKhncfI= github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4Y= github.com/cosmos/gorocksdb v1.2.0/go.mod h1:aaKvKItm514hKfNJpUJXnnOWeBnk2GL4+Qw9NHizILw= github.com/cosmos/iavl v0.19.4 h1:t82sN+Y0WeqxDLJRSpNd8YFX5URIrT+p8n6oJbJ2Dok= @@ -582,8 +583,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -613,6 +615,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= github.com/google/go-licenses v0.0.0-20210329231322-ce1d9163b77d/go.mod h1:+TYOmkVoJOpwnS0wfdsJCV9CoD5nJYsHoFk/0CrTK4M= @@ -702,7 +705,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4G github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.0.1/go.mod h1:oVMjMN64nzEcepv1kdZKgx1qNYt4Ro0Gqefiq2JWdis= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.0 h1:1JYBfzqrWPcCclBwxFCPAou9n+q86mfnu7NAeHfte7A= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 h1:gDLXvp5S9izjldquuoAhDzccbskOL6tDC5jMSyx3zxE= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/gtank/merlin v0.1.1-0.20191105220539-8318aed1a79f/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= @@ -922,8 +925,8 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= @@ -1795,8 +1798,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20230111222715-75897c7a292a h1:/YWeLOBWYV5WAQORVPkZF3Pq9IppkcT72GKnWjNf5W8= -golang.org/x/exp v0.0.0-20230111222715-75897c7a292a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230131160201-f062dba9d201 h1:BEABXpNXLEz0WxtA+6CQIz2xkg80e+1zrhWyMcq8VzE= +golang.org/x/exp v0.0.0-20230131160201-f062dba9d201/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1903,8 +1906,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2170,7 +2173,6 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2285,8 +2287,8 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210426193834-eac7f76ac494/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210510173355-fb37daa5cd7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 h1:jmIfw8+gSvXcZSgaFAGyInDXeWzUhvYH57G/5GKMn70= -google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 h1:znp6mq/drrY+6khTAlJUDNFFcDGV2ENLYKpMq8SyCds= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -2320,8 +2322,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ= -google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/model/const.go b/model/const.go index c5b3c3a2f..2ce611215 100644 --- a/model/const.go +++ b/model/const.go @@ -112,28 +112,45 @@ const ( // define gateway constants const ( - // StreamBufSize defines gateway stream forward payload buf size - StreamBufSize = 64 * 1024 + // DefaultStreamBufSize defines gateway stream forward payload buf size + DefaultStreamBufSize = 64 * 1024 + // DefaultExpiredHeight defines approval expired height + DefaultExpiredHeight = 100 ) -// http header constants +// define downloader constants const ( + // DefaultReadQuotaSize defines bucket's default quota size + DefaultReadQuotaSize = 10 * 1024 * 1024 * 1024 +) - // ContentTypeHeader and below are standard http protocols - ContentTypeHeader = "Content-Type" - ETagHeader = "ETag" - ContentTypeXMLHeaderValue = "application/xml" - RangeHeader = "Range" - ContentRangeHeader = "Content-Range" - OctetStream = "application/octet-stream" +// http header constants +const ( + // ContentTypeHeader is used to indicate the media type of the resource + ContentTypeHeader = "Content-Type" + // ETagHeader is an MD5 digest of the object data + ETagHeader = "ETag" + // RangeHeader asks the server to send only a portion of an HTTP message back to a client + RangeHeader = "Range" + // ContentRangeHeader response HTTP header indicates where in a full body message a partial message belongs + ContentRangeHeader = "Content-Range" + // OctetStream is used to indicate the binary files + OctetStream = "application/octet-stream" + // ContentTypeJSONHeaderValue is used to indicate json ContentTypeJSONHeaderValue = "application/json" + // ContentTypeXMLHeaderValue is used to indicate xml + ContentTypeXMLHeaderValue = "application/xml" - // SignAlgorithm and below are the signature-related constants + // SignAlgorithm uses secp256k1 with the ECDSA algorithm SignAlgorithm = "ECDSA-secp256k1" - SignedMsg = "SignedMsg" - Signature = "Signature" - SignTypeV1 = "authTypeV1" - SignTypeV2 = "authTypeV2" + // SignedMsg is the request hash + SignedMsg = "SignedMsg" + // Signature is the request signature + Signature = "Signature" + // SignTypeV1 is an authentication algorithm, which is used by dapps + SignTypeV1 = "authTypeV1" + // SignTypeV2 is an authentication algorithm, which is used by metamask + SignTypeV2 = "authTypeV2" // GetApprovalPath defines get-approval path style suffix GetApprovalPath = "/greenfield/admin/v1/get-approval" @@ -165,8 +182,8 @@ const ( GnfdSignedApprovalMsgHeader = "X-Gnfd-Signed-Msg" // GnfdObjectInfoHeader define object info, which is used by syncer GnfdObjectInfoHeader = "X-Gnfd-Object-Info" - // GnfdReplicateIdxHeader defines replicate idx, which is used by syncer - GnfdReplicateIdxHeader = "X-Gnfd-Replicate-Idx" + // GnfdReplicaIdxHeader defines replica idx, which is used by syncer + GnfdReplicaIdxHeader = "X-Gnfd-Replica-Idx" // GnfdSegmentSizeHeader defines segment size, which is used by syncer GnfdSegmentSizeHeader = "X-Gnfd-Segment-Size" // GnfdIntegrityHashSignatureHeader defines integrity hash signature, which is used by syncer diff --git a/model/errors/rpc_error.go b/model/errors/rpc_error.go index d0958f102..7530496fb 100644 --- a/model/errors/rpc_error.go +++ b/model/errors/rpc_error.go @@ -10,6 +10,10 @@ var ( ErrCacheMiss = errors.New("cache missing") // ErrSealTimeout defines seal object timeout error ErrSealTimeout = errors.New("seal object timeout") + // ErrNoSuchObject defines not existed object error + ErrNoSuchObject = errors.New("the specified key does not exist") + // ErrNoSuchBucket defines not existed bucket error + ErrNoSuchBucket = errors.New("the specified bucket does not exist") ) // piece store errors @@ -20,10 +24,6 @@ var ( ErrUnsupportedDelimiter = errors.New("unsupported delimiter") // ErrInvalidObjectKey defines invalid object key error ErrInvalidObjectKey = errors.New("invalid object key") - // ErrNotExitObject defines not exist object in memory error - ErrNotExitObject = errors.New("object not exist") - // ErrNotExistBucket defines not exist bucket error - ErrNotExistBucket = errors.New("bucket not exist") // ErrNoPermissionAccessBucket defines deny access bucket error ErrNoPermissionAccessBucket = errors.New("deny access bucket") ) @@ -38,19 +38,28 @@ var ( ErrDuplicateObject = errors.New("duplicate object") // ErrPayloadZero defines payload size is zero error ErrPayloadZero = errors.New("object payload is zero") - // ErrAuthorizationFormat defines the invalid authorization format error ErrAuthorizationFormat = errors.New("authorization format error") // ErrRequestConsistent defines the invalid request checksum error - ErrRequestConsistent = errors.New("failed to check request consistent") + ErrRequestConsistent = errors.New("request is tampered") // ErrSignatureConsistent defines the invalid signature error - ErrSignatureConsistent = errors.New("failed to check signature consistent") + ErrSignatureConsistent = errors.New("signature is not consistent") // ErrUnsupportedSignType defines the unsupported signature type error ErrUnsupportedSignType = errors.New("unsupported signature type") // ErrEmptyReqHeader defines the empty header error ErrEmptyReqHeader = errors.New("request header is empty") // ErrInvalidHeader defines the invalid header error ErrInvalidHeader = errors.New("invalid request header") + // ErrNoPermission defines the authorization error + ErrNoPermission = errors.New("no permission") + // ErrCheckObjectCreated defines the check object state error + ErrCheckObjectCreated = errors.New("object is not created") + // ErrCheckObjectSealed defines the check object state error + ErrCheckObjectSealed = errors.New("object is not sealed") + // ErrCheckPaymentAccountActive defines check payment account state is active + ErrCheckPaymentAccountActive = errors.New("payment account is not active") + // ErrCheckQuotaEnough defines check quota is enough + ErrCheckQuotaEnough = errors.New("quota is not enough") ) // signer service error diff --git a/pkg/greenfield/auth_service.go b/pkg/greenfield/auth_service.go deleted file mode 100644 index 5bee74018..000000000 --- a/pkg/greenfield/auth_service.go +++ /dev/null @@ -1,127 +0,0 @@ -package greenfield - -import ( - "context" - - storagetypes "github.com/bnb-chain/greenfield/x/storage/types" - - "github.com/bnb-chain/greenfield-storage-provider/pkg/log" -) - -// AuthUploadObjectWithAccount verify the greenfield chain information for upload object. -func (greenfield *Greenfield) AuthUploadObjectWithAccount(ctx context.Context, bucket, object, account, sp string) ( - accountExist bool, bucketExist bool, isInitStatus bool, paymentEnough bool, - isSpBucket bool, ownerObject bool, err error) { - accountExist, err = greenfield.HasAccount(ctx, account) - if err != nil || !accountExist { - log.Errorw("failed to query account", "bucket", bucket, - "object", object, "account_exist", accountExist, "error", err) - return - } - var bucketInfo *storagetypes.BucketInfo - bucketInfo, err = greenfield.QueryBucketInfo(ctx, bucket) - if err != nil || bucketInfo == nil { - bucketExist = false - log.Errorw("failed to query bucket info", "bucket", bucket, - "object", object, "error", err) - return - } - bucketExist = true - - var objectInfo *storagetypes.ObjectInfo - objectInfo, err = greenfield.QueryObjectInfo(ctx, bucket, object) - if err != nil || objectInfo == nil { - isInitStatus = false - log.Errorw("failed to query object info", "bucket", bucket, - "object", object, "error", err) - return - } - if objectInfo.GetObjectStatus() != storagetypes.OBJECT_STATUS_INIT { - isInitStatus = false - log.Errorw("object status is not equal to status_init", - "status", objectInfo.GetObjectStatus()) - return - } - isInitStatus = true - if objectInfo.GetOwner() != account { - ownerObject = false - log.Errorw("object owner is not equal to account", - "owner", objectInfo.GetOwner(), "account", account) - return - } - ownerObject = true - - // TODO:: check payment address whether in arrears status - paymentEnough = true - // TODO: sp should be operator address - /* - if bucketInfo.GetPrimarySpAddress() == sp { - spBucket = true - } else { - log.Errorw("object sp is not equal to primary sp", "owner_sp", bucketInfo.GetPrimarySpAddress(), "sp", sp) - spBucket = false - } - */ - isSpBucket = true - return -} - -// AuthDownloadObjectWithAccount verify the greenfield chain information for download object. -func (greenfield *Greenfield) AuthDownloadObjectWithAccount(ctx context.Context, bucket, object, account, sp string) ( - accountExist bool, bucketExist bool, isServiceStatus bool, paymentEnough bool, - isSpBucket bool, bucketID uint64, readQuota int32, ownerObject bool, err error) { - - accountExist, err = greenfield.HasAccount(ctx, account) - if err != nil || !accountExist { - log.Errorw("failed to query account", "bucket", bucket, - "object", object, "account_exist", accountExist, "error", err) - return - } - var bucketInfo *storagetypes.BucketInfo - bucketInfo, err = greenfield.QueryBucketInfo(ctx, bucket) - if err != nil || bucketInfo == nil { - bucketExist = false - log.Errorw("failed to query bucket info", - "bucket", bucket, "object", object, "error", err) - return - } - bucketExist = true - - var objectInfo *storagetypes.ObjectInfo - objectInfo, err = greenfield.QueryObjectInfo(ctx, bucket, object) - if err != nil || objectInfo == nil { - isServiceStatus = false - log.Errorw("failed to query object info", - "bucket", bucket, "object", object, "error", err) - return - } - if objectInfo.GetObjectStatus() != storagetypes.OBJECT_STATUS_IN_SERVICE { - isServiceStatus = false - log.Errorw("object not in status_in_service", - "status", objectInfo.GetObjectStatus()) - return - } - isServiceStatus = true - if objectInfo.GetOwner() != account { - ownerObject = false - log.Errorw("object owner mismatch", "owner", objectInfo.GetOwner(), "account", account) - return - } - ownerObject = true - - // TODO:: check payment address whether in arrears status - paymentEnough = true - // TODO: sp should be operator address - /* - if bucketInfo.GetPrimarySpAddress() == sp { - spBucket = true - } else { - spBucket = false - log.Errorw("object sp is not equal to primary sp", "owner_sp", bucketInfo.GetPrimarySpAddress(), "sp", sp) - } - */ - isSpBucket = true - bucketID = bucketInfo.Id.Uint64() - readQuota = int32(bucketInfo.GetReadQuota()) - return -} diff --git a/pkg/greenfield/query_service.go b/pkg/greenfield/query_service.go index 21a1643d2..aa63954d1 100644 --- a/pkg/greenfield/query_service.go +++ b/pkg/greenfield/query_service.go @@ -2,16 +2,17 @@ package greenfield import ( "context" + "errors" "math" "time" + paymenttypes "github.com/bnb-chain/greenfield/x/payment/types" sptypes "github.com/bnb-chain/greenfield/x/sp/types" storagetypes "github.com/bnb-chain/greenfield/x/storage/types" - "github.com/cosmos/cosmos-sdk/types/query" authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" - merror "github.com/bnb-chain/greenfield-storage-provider/model/errors" + merrors "github.com/bnb-chain/greenfield-storage-provider/model/errors" "github.com/bnb-chain/greenfield-storage-provider/pkg/log" ) @@ -96,6 +97,19 @@ func (greenfield *Greenfield) QueryObjectInfo(ctx context.Context, bucket, objec return resp.GetObjectInfo(), nil } +// QueryBucketInfoAndObjectInfo return bucket info and object info, if not found, return the corresponding error code +func (greenfield *Greenfield) QueryBucketInfoAndObjectInfo(ctx context.Context, bucket, object string) (*storagetypes.BucketInfo, *storagetypes.ObjectInfo, error) { + bucketInfo, err := greenfield.QueryBucketInfo(ctx, bucket) + if errors.Is(err, storagetypes.ErrNoSuchBucket) { + return nil, nil, merrors.ErrNoSuchBucket + } + objectInfo, err := greenfield.QueryObjectInfo(ctx, bucket, object) + if errors.Is(err, storagetypes.ErrNoSuchObject) { + return nil, nil, merrors.ErrNoSuchObject + } + return bucketInfo, objectInfo, nil +} + // ListenObjectSeal return an indication of the object is sealed. // TODO:: retrieve service support seal event subscription func (greenfield *Greenfield) ListenObjectSeal(ctx context.Context, bucket, object string, timeOutHeight int) (seal bool, err error) { @@ -106,13 +120,26 @@ func (greenfield *Greenfield) ListenObjectSeal(ctx context.Context, bucket, obje if err != nil { continue } - if objectInfo.GetObjectStatus() == storagetypes.OBJECT_STATUS_IN_SERVICE { + if objectInfo.GetObjectStatus() == storagetypes.OBJECT_STATUS_SEALED { seal = true err = nil return } } log.Errorw("seal object timeout", "bucket_name", bucket, "object_name", object) - err = merror.ErrSealTimeout + err = merrors.ErrSealTimeout return } + +// QueryStreamRecord return the steam record info by account. +func (greenfield *Greenfield) QueryStreamRecord(ctx context.Context, account string) (*paymenttypes.StreamRecord, error) { + client := greenfield.getCurrentClient().GnfdCompositeClient() + resp, err := client.StreamRecord(ctx, &paymenttypes.QueryGetStreamRecordRequest{ + Account: account, + }) + if err != nil { + log.Errorw("failed to query stream record", "account", account, "error", err) + return nil, err + } + return &resp.StreamRecord, nil +} diff --git a/pkg/stream/payload_stream.go b/pkg/stream/payload_stream.go index 322316e81..24325cefc 100644 --- a/pkg/stream/payload_stream.go +++ b/pkg/stream/payload_stream.go @@ -28,7 +28,7 @@ func (entry SegmentEntry) ID() uint64 { func (entry SegmentEntry) Key() string { if entry.redundancyType == storagetypes.REDUNDANCY_EC_TYPE { - return piecestore.EncodeECPieceKey(entry.objectID, entry.replicaIdx, entry.segmentIdx) + return piecestore.EncodeECPieceKey(entry.objectID, entry.segmentIdx, entry.replicaIdx) } return piecestore.EncodeSegmentPieceKey(entry.objectID, entry.segmentIdx) } diff --git a/proto/service/challenge/types/challenge.proto b/proto/service/challenge/types/challenge.proto index 170de89ba..9e14f3614 100644 --- a/proto/service/challenge/types/challenge.proto +++ b/proto/service/challenge/types/challenge.proto @@ -7,10 +7,10 @@ option go_package = "github.com/bnb-chain/greenfield-storage-provider/service/ch message ChallengePieceRequest { // object_id defines the challenge object id uint64 object_id = 1; - // replicate_idx defines the challenge replicate idx - int32 replicate_idx = 2; + // replica_idx defines the challenge replica idx + int32 replica_idx = 2; // segment_idx defines the challenge segment idx - uint32 segment_idx = 5; + uint32 segment_idx = 3; } // ChallengePieceResponse is response type for the ChallengePiece RPC method. diff --git a/proto/service/downloader/types/downloader.proto b/proto/service/downloader/types/downloader.proto index 2d8aa2ed5..60dc29e43 100644 --- a/proto/service/downloader/types/downloader.proto +++ b/proto/service/downloader/types/downloader.proto @@ -1,31 +1,35 @@ syntax = "proto3"; package service.downloader.types; +import "greenfield/storage/types.proto"; + option go_package = "github.com/bnb-chain/greenfield-storage-provider/service/downloader/types"; -// DownloaderObjectRequest is request type for the DownloaderObject RPC method. -message DownloaderObjectRequest { - // bucket_name defines the download bucket name - string bucket_name = 1; - // object_name defines the download object name - string object_name = 2; +// GetObjectRequest is request type for the GetObject RPC method. +message GetObjectRequest { + // bucket_info defines the information of the bucket. + bnbchain.greenfield.storage.BucketInfo bucket_info = 1; + // object_info defines the information of the object. + bnbchain.greenfield.storage.ObjectInfo object_info = 2; + // user_address defines the request user address. + string user_address = 3; // offset defines the download payload offset - uint64 offset = 3; + uint64 offset = 4; // length defines the download payload length - uint64 length = 4; // if length == 0, download all object data + uint64 length = 5; // if length == 0, download all object data // is_range indicate whether it is rang download, compatible aws s3 - bool is_range = 5; + bool is_range = 6; // range_start defines the start of range // [range_start, range_end], range_start >= object_size is invalid, download all object data - int64 range_start = 6; + int64 range_start = 7; // range_end defines the end of range // range_end >= object_size is invalid, download all object data // range_end < 0, download [range_start, object_end] - int64 range_end = 7; + int64 range_end = 8; } -// DownloaderObjectResponse is response type for the DownloaderObject RPC method. -message DownloaderObjectResponse { +// GetObjectResponse is response type for the GetObject RPC method. +message GetObjectResponse { // is_valid_range indicates whether ths range valid bool is_valid_range = 1; // data defines the download data @@ -34,6 +38,6 @@ message DownloaderObjectResponse { // DownloaderService defines the gRPC service of download payload. service DownloaderService { - // DownloaderObject download the payload of the object. - rpc DownloaderObject(DownloaderObjectRequest) returns (stream DownloaderObjectResponse) {}; + // GetObject download the payload of the object. + rpc GetObject(GetObjectRequest) returns (stream GetObjectResponse) {}; } diff --git a/proto/service/syncer/types/syncer_service.proto b/proto/service/syncer/types/syncer_service.proto index 0ed3b3d3c..a96bbdfa0 100644 --- a/proto/service/syncer/types/syncer_service.proto +++ b/proto/service/syncer/types/syncer_service.proto @@ -10,12 +10,12 @@ option go_package = "github.com/bnb-chain/greenfield-storage-provider/service/sy message SyncObjectRequest { // object_info defines the information of the object. bnbchain.greenfield.storage.ObjectInfo object_info = 1; - // replicate_idx defines the index of replicate. - uint32 replicate_idx = 2; + // replica_idx defines the index of replicate. + uint32 replica_idx = 2; // segment_size defines the spilt size of replicate data. uint64 segment_size = 3; - // replicate_data defines the data that replicate to storage provider. - bytes replicate_data = 4; + // replica_data defines the data that replicate to storage provider. + bytes replica_data = 4; } // SyncObjectResponse is response type for the SyncObject RPC method. diff --git a/proto/service/uploader/types/uploader_service.proto b/proto/service/uploader/types/uploader_service.proto index 18b2e3527..20b216096 100644 --- a/proto/service/uploader/types/uploader_service.proto +++ b/proto/service/uploader/types/uploader_service.proto @@ -6,34 +6,34 @@ import "service/types/storage_provider.proto"; option go_package = "github.com/bnb-chain/greenfield-storage-provider/service/uploader/types"; -// UploadObjectRequest is request type for the UploadObject RPC method. -message UploadObjectRequest { +// PutObjectRequest is request type for the UploadObject RPC method. +message PutObjectRequest { // object_info defines the information of the object. bnbchain.greenfield.storage.ObjectInfo object_info = 1; // payload defines the data of the object. bytes payload = 2; } -// UploadObjectResponse is response type for the UploadObject RPC method. -message UploadObjectResponse { +// PutObjectResponse is response type for the UploadObject RPC method. +message PutObjectResponse { } -// QueryUploadingObjectRequest is request type for the QueryUploadingObject RPC method. -message QueryUploadingObjectRequest { +// QueryUPuttingObjectRequest is request type for the QueryPuttingObject RPC method. +message QueryPuttingObjectRequest { // object_id defines the unique id of the object. uint64 object_id = 1; } -// QueryUploadingObjectResponse is response type for the QueryUploadingObject RPC method. -message QueryUploadingObjectResponse { +// QueryPuttingObjectResponse is response type for the QueryPuttingObject RPC method. +message QueryPuttingObjectResponse { // segment_info defines the information of uploading object. service.types.SegmentInfo segment_info = 1; } // UploaderService defines the gRPC service of uploading payload. service UploaderService { - // UploadObject upload the payload of the object. - rpc UploadObject(stream UploadObjectRequest) returns (UploadObjectResponse) {}; - // QueryUploadingObject query an uploading object info with object id. - rpc QueryUploadingObject(QueryUploadingObjectRequest) returns (QueryUploadingObjectResponse) {}; + // PutObject upload the payload of the object. + rpc PutObject(stream PutObjectRequest) returns (PutObjectResponse) {}; + // QueryPuttingObject query an uploading object info with object id. + rpc QueryPuttingObject(QueryPuttingObjectRequest) returns (QueryPuttingObjectResponse) {}; } \ No newline at end of file diff --git a/service/challenge/challenge_service.go b/service/challenge/challenge_service.go index d0a4a60fc..4cb265e26 100644 --- a/service/challenge/challenge_service.go +++ b/service/challenge/challenge_service.go @@ -26,11 +26,11 @@ func (challenge *Challenge) ChallengePiece( resp.PieceHash = integrity.Checksum var key string - if req.GetReplicateIdx() < 0 { + if req.GetReplicaIdx() < 0 { key = piecestore.EncodeSegmentPieceKey(req.GetObjectId(), req.GetSegmentIdx()) } else { key = piecestore.EncodeECPieceKey(req.GetObjectId(), - uint32(req.GetReplicateIdx()), req.GetSegmentIdx()) + req.GetSegmentIdx(), uint32(req.GetReplicaIdx())) } resp.PieceData, err = challenge.pieceStore.GetSegment(ctx, key, 0, -1) if err != nil { diff --git a/service/challenge/client/challenge_client.go b/service/challenge/client/challenge_client.go index 60c9ce353..873a96565 100644 --- a/service/challenge/client/challenge_client.go +++ b/service/challenge/client/challenge_client.go @@ -39,12 +39,12 @@ func (client *ChallengeClient) Close() error { } // ChallengePiece send challenge piece request -func (client *ChallengeClient) ChallengePiece(ctx context.Context, objectID uint64, replicateIdx int32, segmentIdx uint32, +func (client *ChallengeClient) ChallengePiece(ctx context.Context, objectID uint64, replicaIdx int32, segmentIdx uint32, opts ...grpc.CallOption) ([]byte, [][]byte, []byte, error) { resp, err := client.challenge.ChallengePiece(ctx, &types.ChallengePieceRequest{ - ObjectId: objectID, - ReplicateIdx: replicateIdx, - SegmentIdx: segmentIdx, + ObjectId: objectID, + ReplicaIdx: replicaIdx, + SegmentIdx: segmentIdx, }, opts...) return resp.IntegrityHash, resp.PieceHash, resp.PieceData, err } diff --git a/service/downloader/client/downloader_client.go b/service/downloader/client/downloader_client.go index b45798e22..07a1836b5 100644 --- a/service/downloader/client/downloader_client.go +++ b/service/downloader/client/downloader_client.go @@ -40,9 +40,9 @@ func (client *DownloaderClient) Close() error { return client.conn.Close() } -// DownloaderObject download the payload of the object -func (client *DownloaderClient) DownloaderObject(ctx context.Context, req *types.DownloaderObjectRequest, - opts ...grpc.CallOption) (types.DownloaderService_DownloaderObjectClient, error) { +// GetObject download the payload of the object +func (client *DownloaderClient) GetObject(ctx context.Context, req *types.GetObjectRequest, + opts ...grpc.CallOption) (types.DownloaderService_GetObjectClient, error) { // ctx = log.Context(context.Background(), req) - return client.downloader.DownloaderObject(ctx, req, opts...) + return client.downloader.GetObject(ctx, req, opts...) } diff --git a/service/downloader/downloader.go b/service/downloader/downloader.go index d01bf21fa..c869f610c 100644 --- a/service/downloader/downloader.go +++ b/service/downloader/downloader.go @@ -9,7 +9,6 @@ import ( "google.golang.org/grpc/reflection" "github.com/bnb-chain/greenfield-storage-provider/model" - gnfd "github.com/bnb-chain/greenfield-storage-provider/pkg/greenfield" "github.com/bnb-chain/greenfield-storage-provider/pkg/log" "github.com/bnb-chain/greenfield-storage-provider/service/downloader/types" psclient "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/client" @@ -20,7 +19,6 @@ import ( type Downloader struct { cfg *DownloaderConfig spDB sqldb.SPDB - chain *gnfd.Greenfield pieceStore *psclient.StoreClient } @@ -32,11 +30,6 @@ func NewDownloaderService(cfg *DownloaderConfig) (*Downloader, error) { log.Errorw("failed to create piece store client", "error", err) return nil, err } - chain, err := gnfd.NewGreenfield(cfg.ChainConfig) - if err != nil { - log.Errorw("failed to create chain client", "error", err) - return nil, err - } spDB, err := sqldb.NewSpDB(cfg.SpDBConfig) if err != nil { log.Errorw("failed to create spdb client", "error", err) @@ -45,7 +38,6 @@ func NewDownloaderService(cfg *DownloaderConfig) (*Downloader, error) { downloader := &Downloader{ cfg: cfg, spDB: spDB, - chain: chain, pieceStore: pieceStore, } return downloader, nil diff --git a/service/downloader/downloader_config.go b/service/downloader/downloader_config.go index 45b923c52..020245c9b 100644 --- a/service/downloader/downloader_config.go +++ b/service/downloader/downloader_config.go @@ -1,7 +1,6 @@ package downloader import ( - gnfd "github.com/bnb-chain/greenfield-storage-provider/pkg/greenfield" "github.com/bnb-chain/greenfield-storage-provider/store/config" "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/storage" ) @@ -10,5 +9,4 @@ type DownloaderConfig struct { GRPCAddress string SpDBConfig *config.SQLDBConfig PieceStoreConfig *storage.PieceStoreConfig - ChainConfig *gnfd.GreenfieldChainConfig } diff --git a/service/downloader/downloader_service.go b/service/downloader/downloader_service.go index dc22068ab..e87f99d5b 100644 --- a/service/downloader/downloader_service.go +++ b/service/downloader/downloader_service.go @@ -4,27 +4,26 @@ import ( "context" "fmt" - storagetypes "github.com/bnb-chain/greenfield/x/storage/types" - + "github.com/bnb-chain/greenfield-storage-provider/model" "github.com/bnb-chain/greenfield-storage-provider/model/piecestore" "github.com/bnb-chain/greenfield-storage-provider/pkg/log" "github.com/bnb-chain/greenfield-storage-provider/service/downloader/types" + "github.com/bnb-chain/greenfield-storage-provider/store/sqldb" ) var _ types.DownloaderServiceServer = &Downloader{} // DownloaderObject download the payload of the object. -func (downloader *Downloader) DownloaderObject(req *types.DownloaderObjectRequest, - stream types.DownloaderService_DownloaderObjectServer) (err error) { +func (downloader *Downloader) GetObject(req *types.GetObjectRequest, + stream types.DownloaderService_GetObjectServer) (err error) { var ( - objectInfo *storagetypes.ObjectInfo size int offset uint64 length uint64 isValidRange bool ) ctx := log.Context(context.Background(), req) - resp := &types.DownloaderObjectResponse{} + resp := &types.GetObjectResponse{} defer func() { if err != nil { return @@ -33,17 +32,28 @@ func (downloader *Downloader) DownloaderObject(req *types.DownloaderObjectReques log.CtxInfow(ctx, "finish to download object", "error", err, "sendSize", size) }() - chainObjectInfo, err := downloader.chain.QueryObjectInfo(ctx, req.BucketName, req.ObjectName) - if err != nil { - log.Errorf("failed to query chain", "err", err) - return - } - objectInfo = &storagetypes.ObjectInfo{ - Id: chainObjectInfo.Id, - PayloadSize: chainObjectInfo.PayloadSize, + bucketInfo := req.GetBucketInfo() + objectInfo := req.GetObjectInfo() + if err = downloader.spDB.CheckQuotaAndAddReadRecord( + // TODO: support range read + &sqldb.ReadRecord{ + BucketID: bucketInfo.Id.Uint64(), + ObjectID: objectInfo.Id.Uint64(), + UserAddress: req.GetUserAddress(), + BucketName: bucketInfo.GetBucketName(), + ObjectName: objectInfo.GetObjectName(), + ReadSize: int64(objectInfo.PayloadSize), + ReadTime: sqldb.GetCurrentUnixTime(), + }, + &sqldb.BucketQuota{ + ReadQuotaSize: int64(bucketInfo.GetReadQuota()) + model.DefaultReadQuotaSize, + }, + ); err != nil { + log.Errorw("failed to check billing due to bucket quota", "error", err) + return err } - // TODO: It will be optimized here after connecting with the chain + // TODO: It will be optimized // if length == 0, download all object data if req.RangeStart >= 0 && req.RangeStart < int64(objectInfo.GetPayloadSize()) && req.RangeEnd >= 0 && req.RangeEnd < int64(objectInfo.GetPayloadSize()) { @@ -57,10 +67,6 @@ func (downloader *Downloader) DownloaderObject(req *types.DownloaderObjectReques } else { offset, length = 0, objectInfo.GetPayloadSize() } - // offset, length = req.GetOffset(), req.GetLength() - // if req.GetLength() == 0 { - // offset, length = 0, objectInfo.Size - // } var segmentInfo segments segmentInfo, err = downloader.DownloadPieceInfo(objectInfo.Id.Uint64(), objectInfo.GetPayloadSize(), offset, offset+length-1) if err != nil { diff --git a/service/gateway/admin_handler.go b/service/gateway/admin_handler.go index da14de699..faa3f658f 100644 --- a/service/gateway/admin_handler.go +++ b/service/gateway/admin_handler.go @@ -3,7 +3,6 @@ package gateway import ( "context" "encoding/hex" - "math" "net/http" "github.com/bnb-chain/greenfield/x/storage/types" @@ -43,12 +42,12 @@ func (g *Gateway) getApprovalHandler(w http.ResponseWriter, r *http.Request) { if addr, err = reqContext.verifySignature(); err != nil { log.Errorw("failed to verify signature", "error", err) - errDescription = SignatureNotMatch + errDescription = makeErrorDescription(err) return } if err = g.checkAuthorization(reqContext, addr); err != nil { log.Errorw("failed to check authorization", "error", err) - errDescription = UnauthorizedAccess + errDescription = makeErrorDescription(err) return } @@ -76,12 +75,11 @@ func (g *Gateway) getApprovalHandler(w http.ResponseWriter, r *http.Request) { errDescription = InvalidHeader return } - // TODO: to config it - msg.PrimarySpApproval = &types.Approval{ExpiredHeight: math.MaxUint64} + msg.PrimarySpApproval = &types.Approval{ExpiredHeight: model.DefaultExpiredHeight} approvalSignature, err = g.signer.SignBucketApproval(context.Background(), &msg) if err != nil { log.Errorw("failed to sign create bucket approval", "error", err) - errDescription = InternalError + errDescription = makeErrorDescription(err) return } msg.PrimarySpApproval.Sig = approvalSignature @@ -102,12 +100,11 @@ func (g *Gateway) getApprovalHandler(w http.ResponseWriter, r *http.Request) { errDescription = InvalidHeader return } - // TODO: to config it - msg.PrimarySpApproval = &types.Approval{ExpiredHeight: math.MaxUint64} + msg.PrimarySpApproval = &types.Approval{ExpiredHeight: model.DefaultExpiredHeight} approvalSignature, err = g.signer.SignObjectApproval(context.Background(), &msg) if err != nil { log.Errorw("failed to sign create object approval", "error", err) - errDescription = InternalError + errDescription = makeErrorDescription(err) return } msg.PrimarySpApproval.Sig = approvalSignature @@ -153,12 +150,12 @@ func (g *Gateway) challengeHandler(w http.ResponseWriter, r *http.Request) { if addr, err = reqContext.verifySignature(); err != nil { log.Errorw("failed to verify signature", "error", err) - errDescription = SignatureNotMatch + errDescription = makeErrorDescription(err) return } if err = g.checkAuthorization(reqContext, addr); err != nil { log.Errorw("failed to check authorization", "error", err) - errDescription = UnauthorizedAccess + errDescription = makeErrorDescription(err) return } @@ -181,7 +178,7 @@ func (g *Gateway) challengeHandler(w http.ResponseWriter, r *http.Request) { integrityHash, pieceHash, pieceData, err := g.challenge.ChallengePiece(context.Background(), objectID, redundancyIdx, segmentIdx) if err != nil { log.Errorf("failed to challenge", "error", err) - errDescription = InternalError + errDescription = makeErrorDescription(err) return } w.Header().Set(model.GnfdRequestIDHeader, reqContext.requestID) diff --git a/service/gateway/client/gateway_client.go b/service/gateway/client/gateway_client.go index b001c12c9..cb3923a13 100644 --- a/service/gateway/client/gateway_client.go +++ b/service/gateway/client/gateway_client.go @@ -89,7 +89,7 @@ func (p *PieceDataReader) Read(buf []byte) (n int, err error) { // SyncPieceData sync piece data to the target storage-provider. func (gatewayClient *GatewayClient) SyncPieceData( objectInfo *types.ObjectInfo, - replicateIdx uint32, + replicaIdx uint32, segmentSize uint32, pieceData [][]byte) (integrityHash []byte, signature []byte, err error) { pieceDataReader, err := NewPieceDataReader(pieceData) @@ -104,7 +104,7 @@ func (gatewayClient *GatewayClient) SyncPieceData( } marshalObjectInfo := hex.EncodeToString(types.ModuleCdc.MustMarshalJSON(objectInfo)) req.Header.Add(model.GnfdObjectInfoHeader, marshalObjectInfo) - req.Header.Add(model.GnfdReplicateIdxHeader, util.Uint32ToString(replicateIdx)) + req.Header.Add(model.GnfdReplicaIdxHeader, util.Uint32ToString(replicaIdx)) req.Header.Add(model.GnfdSegmentSizeHeader, util.Uint32ToString(segmentSize)) req.Header.Add(model.ContentTypeHeader, model.OctetStream) diff --git a/service/gateway/gateway.go b/service/gateway/gateway.go index d2c77d40b..be6816e6e 100644 --- a/service/gateway/gateway.go +++ b/service/gateway/gateway.go @@ -11,8 +11,9 @@ import ( "github.com/gorilla/mux" "github.com/bnb-chain/greenfield-storage-provider/model" - chainclient "github.com/bnb-chain/greenfield-storage-provider/pkg/greenfield" "github.com/bnb-chain/greenfield-storage-provider/pkg/log" + + chainclient "github.com/bnb-chain/greenfield-storage-provider/pkg/greenfield" challengeclient "github.com/bnb-chain/greenfield-storage-provider/service/challenge/client" downloaderclient "github.com/bnb-chain/greenfield-storage-provider/service/downloader/client" signerclient "github.com/bnb-chain/greenfield-storage-provider/service/signer/client" diff --git a/service/gateway/gateway_config.go b/service/gateway/gateway_config.go index 649feb241..a72e0ce25 100644 --- a/service/gateway/gateway_config.go +++ b/service/gateway/gateway_config.go @@ -8,11 +8,11 @@ type GatewayConfig struct { SpOperatorAddress string HTTPAddress string Domain string + ChainConfig *gnfd.GreenfieldChainConfig UploaderServiceAddress string DownloaderServiceAddress string SignerServiceAddress string ChallengeServiceAddress string SyncerServiceAddress string - ChainConfig *gnfd.GreenfieldChainConfig MetadataServiceAddress string } diff --git a/service/gateway/object_handler.go b/service/gateway/object_handler.go index 42d0fffd4..1a64ec1b4 100644 --- a/service/gateway/object_handler.go +++ b/service/gateway/object_handler.go @@ -7,7 +7,7 @@ import ( "io" "net/http" - storagetypes "github.com/bnb-chain/greenfield/x/storage/types" + "github.com/bnb-chain/greenfield/types/s3util" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/bnb-chain/greenfield-storage-provider/model" @@ -50,12 +50,12 @@ func (g *Gateway) getObjectHandler(w http.ResponseWriter, r *http.Request) { return } - if err = storagetypes.CheckValidBucketName(reqContext.bucketName); err != nil { + if err = s3util.CheckValidBucketName(reqContext.bucketName); err != nil { log.Errorw("failed to check bucket name", "bucket_name", reqContext.bucketName, "error", err) errDescription = InvalidBucketName return } - if err = storagetypes.CheckValidObjectName(reqContext.objectName); err != nil { + if err = s3util.CheckValidObjectName(reqContext.objectName); err != nil { log.Errorw("failed to check object name", "object_name", reqContext.objectName, "error", err) errDescription = InvalidKey return @@ -63,12 +63,12 @@ func (g *Gateway) getObjectHandler(w http.ResponseWriter, r *http.Request) { if addr, err = reqContext.verifySignature(); err != nil { log.Errorw("failed to verify signature", "error", err) - errDescription = SignatureNotMatch + errDescription = makeErrorDescription(err) return } if err = g.checkAuthorization(reqContext, addr); err != nil { log.Errorw("failed to check authorization", "error", err) - errDescription = UnauthorizedAccess + errDescription = makeErrorDescription(err) return } @@ -79,18 +79,19 @@ func (g *Gateway) getObjectHandler(w http.ResponseWriter, r *http.Request) { return } - req := &types.DownloaderObjectRequest{ - BucketName: reqContext.bucketName, - ObjectName: reqContext.objectName, - IsRange: isRange, - RangeStart: rangeStart, - RangeEnd: rangeEnd, + req := &types.GetObjectRequest{ + BucketInfo: reqContext.bucketInfo, + ObjectInfo: reqContext.objectInfo, + UserAddress: addr.String(), + IsRange: isRange, + RangeStart: rangeStart, + RangeEnd: rangeEnd, } ctx := log.Context(context.Background(), req) - stream, err := g.downloader.DownloaderObject(ctx, req) + stream, err := g.downloader.GetObject(ctx, req) if err != nil { log.Errorf("failed to get object", "error", err) - errDescription = InternalError + errDescription = makeErrorDescription(err) return } for { @@ -100,11 +101,10 @@ func (g *Gateway) getObjectHandler(w http.ResponseWriter, r *http.Request) { } if err != nil { log.Errorw("failed to read stream", "error", err) - errDescription = InternalError + errDescription = makeErrorDescription(err) return } - // TODO: check error resp.code if readN = len(resp.Data); readN == 0 { log.Errorw("failed to download due to return empty data", "response", resp) continue @@ -116,7 +116,7 @@ func (g *Gateway) getObjectHandler(w http.ResponseWriter, r *http.Request) { } if writeN, err = w.Write(resp.Data); err != nil { log.Errorw("failed to read stream", "error", err) - errDescription = InternalError + errDescription = makeErrorDescription(err) return } if readN != writeN { @@ -138,8 +138,8 @@ func (g *Gateway) putObjectHandler(w http.ResponseWriter, r *http.Request) { addr sdk.AccAddress size int readN int - buf = make([]byte, model.StreamBufSize) - hashBuf = make([]byte, model.StreamBufSize) + buf = make([]byte, model.DefaultStreamBufSize) + hashBuf = make([]byte, model.DefaultStreamBufSize) md5Hash = md5.New() ) @@ -161,12 +161,12 @@ func (g *Gateway) putObjectHandler(w http.ResponseWriter, r *http.Request) { return } - if err = storagetypes.CheckValidBucketName(reqContext.bucketName); err != nil { + if err = s3util.CheckValidBucketName(reqContext.bucketName); err != nil { log.Errorw("failed to check bucket name", "bucket_name", reqContext.bucketName, "error", err) errDescription = InvalidBucketName return } - if err = storagetypes.CheckValidObjectName(reqContext.objectName); err != nil { + if err = s3util.CheckValidObjectName(reqContext.objectName); err != nil { log.Errorw("failed to check object name", "object_name", reqContext.objectName, "error", err) errDescription = InvalidKey return @@ -176,19 +176,19 @@ func (g *Gateway) putObjectHandler(w http.ResponseWriter, r *http.Request) { if addr, err = reqContext.verifySignature(); err != nil { log.Errorw("failed to verify signature", "error", err) - errDescription = SignatureNotMatch + errDescription = makeErrorDescription(err) return } if err = g.checkAuthorization(reqContext, addr); err != nil { log.Errorw("failed to check authorization", "error", err) - errDescription = UnauthorizedAccess + errDescription = makeErrorDescription(err) return } - stream, err := g.uploader.UploadObject(context.Background()) + stream, err := g.uploader.PutObject(context.Background()) if err != nil { log.Errorf("failed to put object", "error", err) - errDescription = InternalError + errDescription = makeErrorDescription(err) return } for { @@ -199,7 +199,7 @@ func (g *Gateway) putObjectHandler(w http.ResponseWriter, r *http.Request) { return } if readN > 0 { - req := &uploadertypes.UploadObjectRequest{ + req := &uploadertypes.PutObjectRequest{ ObjectInfo: reqContext.objectInfo, Payload: buf[:readN], } @@ -221,7 +221,7 @@ func (g *Gateway) putObjectHandler(w http.ResponseWriter, r *http.Request) { _, err = stream.CloseAndRecv() if err != nil { log.Errorw("failed to put object due to stream close", "error", err) - errDescription = InternalError + errDescription = makeErrorDescription(err) return } // succeed to put object diff --git a/service/gateway/request_util.go b/service/gateway/request_util.go index 0cd015514..b6f2a9b7b 100644 --- a/service/gateway/request_util.go +++ b/service/gateway/request_util.go @@ -10,6 +10,7 @@ import ( commonhttp "github.com/bnb-chain/greenfield-common/go/http" signer "github.com/bnb-chain/greenfield-go-sdk/keys/signer" + paymenttypes "github.com/bnb-chain/greenfield/x/payment/types" storagetypes "github.com/bnb-chain/greenfield/x/storage/types" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/ethereum/go-ethereum/crypto/secp256k1" @@ -29,9 +30,9 @@ type requestContext struct { request *http.Request startTime time.Time vars map[string]string - // for auth v2 test - skipAuth bool - // objectInfo is queried from the greenfield blockchain + // TODO: for auth v2 test, remove it in the future + skipAuth bool + bucketInfo *storagetypes.BucketInfo objectInfo *storagetypes.ObjectInfo // accountID is used to provide authentication to the sp accountID string @@ -217,7 +218,7 @@ func parseRange(rangeStr string) (bool, int64, int64) { return false, -1, -1 } -// TODO: can be optimized by retirver +// TODO: can be optimized by retriever // checkAuthorization check addr authorization func (g *Gateway) checkAuthorization(reqContext *requestContext, addr sdk.AccAddress) error { var ( @@ -234,52 +235,68 @@ func (g *Gateway) checkAuthorization(reqContext *requestContext, addr sdk.AccAdd } if !accountExist { log.Errorw("account is not exist", "address", addr.String(), "error", err) - return fmt.Errorf("account is not exist") + return errors.ErrNoPermission } switch mux.CurrentRoute(reqContext.request).GetName() { case putObjectRouterName: - reqContext.objectInfo, err = g.chain.QueryObjectInfo(context.Background(), - reqContext.bucketName, reqContext.objectName) - if err != nil || reqContext.objectInfo == nil { - log.Errorw("failed to query object info on chain", + if reqContext.bucketInfo, reqContext.objectInfo, err = g.chain.QueryBucketInfoAndObjectInfo( + context.Background(), reqContext.bucketName, reqContext.objectName); err != nil { + log.Errorw("failed to query bucket info and object info on chain", "bucket_name", reqContext.bucketName, "object_name", reqContext.objectName, "error", err) return err } - if reqContext.objectInfo.GetObjectStatus() != storagetypes.OBJECT_STATUS_INIT { - log.Errorw("failed to auth due to object status is not init", + if reqContext.objectInfo.GetObjectStatus() != storagetypes.OBJECT_STATUS_CREATED { + log.Errorw("failed to auth due to object status is not created", "object_status", reqContext.objectInfo.GetObjectStatus()) - return fmt.Errorf("account has no permission") + return errors.ErrCheckObjectCreated } if reqContext.objectInfo.GetOwner() != addr.String() { log.Errorw("failed to auth due to account is not equal to object owner", - "object_status", reqContext.objectInfo.GetObjectStatus()) - return fmt.Errorf("account has no permission") + "object_owner", reqContext.objectInfo.GetOwner(), + "request_address", addr.String()) + return errors.ErrNoPermission + } + if reqContext.bucketInfo.GetPrimarySpAddress() != g.config.SpOperatorAddress { + log.Errorw("failed to auth due to bucket primary sp is not equal to current sp", + "bucket_primary_sp", reqContext.bucketInfo.GetPrimarySpAddress(), + "current_sp", g.config.SpOperatorAddress) + return errors.ErrNoPermission } - // TODO: check SP operator address and account payment case getObjectRouterName: - _, bucketExist, isServiceStatus, tokenEnough, isSpBucket, bucketID, readQuota, ownObj, err := g.chain.AuthDownloadObjectWithAccount( - context.Background(), - reqContext.bucketName, - reqContext.objectName, - addr.String(), - g.config.SpOperatorAddress) + if reqContext.bucketInfo, reqContext.objectInfo, err = g.chain.QueryBucketInfoAndObjectInfo( + context.Background(), reqContext.bucketName, reqContext.objectName); err != nil { + log.Errorw("failed to query bucket info and object info on chain", + "bucket_name", reqContext.bucketName, "object_name", reqContext.objectName, "error", err) + return err + } + if reqContext.objectInfo.GetObjectStatus() != storagetypes.OBJECT_STATUS_SEALED { + log.Errorw("object is not sealed", + "status", reqContext.objectInfo.GetObjectStatus()) + return errors.ErrCheckObjectSealed + } + if reqContext.objectInfo.GetOwner() != addr.String() { + log.Errorw("failed to auth due to account is not equal to object owner", + "object_owner", reqContext.objectInfo.GetOwner(), + "request_address", addr.String()) + return errors.ErrNoPermission + } + if reqContext.bucketInfo.GetPrimarySpAddress() != g.config.SpOperatorAddress { + log.Errorw("failed to auth due to bucket primary sp is not equal to current sp", + "bucket_primary_sp", reqContext.bucketInfo.GetPrimarySpAddress(), + "current_sp", g.config.SpOperatorAddress) + return errors.ErrNoPermission + } + streamRecord, err := g.chain.QueryStreamRecord(context.Background(), reqContext.bucketInfo.PaymentAddress) if err != nil { - log.Errorw("failed to auth download", - "bucket_name", reqContext.bucketName, "object_name", reqContext.objectName, - "address", addr.String(), "error", err) + log.Errorw("failed to check billing", "error", err) return err } - if !bucketExist || !isServiceStatus || !tokenEnough || !isSpBucket || !ownObj { - log.Errorw("failed to auth download", - "bucket_name", reqContext.bucketName, "object_name", reqContext.objectName, - "address", addr.String(), "error", err) - return fmt.Errorf("account has no permission") + if streamRecord.Status != paymenttypes.STREAM_ACCOUNT_STATUS_ACTIVE { + log.Errorw("failed to check payment due to account status is not active", "status", streamRecord.Status) + return errors.ErrCheckPaymentAccountActive } - // TODO: query read quota enough - _, _ = bucketID, readQuota - // TODO: update read quota } return nil } diff --git a/service/gateway/response_util.go b/service/gateway/response_util.go index 3b371d05d..4b48e8e12 100644 --- a/service/gateway/response_util.go +++ b/service/gateway/response_util.go @@ -6,6 +6,7 @@ import ( "net/http" "github.com/bnb-chain/greenfield-storage-provider/model" + merrors "github.com/bnb-chain/greenfield-storage-provider/model/errors" "github.com/bnb-chain/greenfield-storage-provider/util" ) @@ -19,19 +20,16 @@ type errorDescription struct { // refer: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html var ( // 4xx - InvalidHeader = &errorDescription{errorCode: "InvalidHeader", errorMessage: "The headers maybe is invalid.", statusCode: http.StatusBadRequest} - InvalidBucketName = &errorDescription{errorCode: "InvalidBucketName", errorMessage: "The specified bucket is not valid.", statusCode: http.StatusBadRequest} - InvalidKey = &errorDescription{errorCode: "InvalidKey", errorMessage: "Object key is Illegal", statusCode: http.StatusBadRequest} - InvalidTxHash = &errorDescription{errorCode: "InvalidTxHash", errorMessage: "transaction hash is Illegal", statusCode: http.StatusBadRequest} - InvalidPayload = &errorDescription{errorCode: "InvalidPayload", errorMessage: "payload is empty", statusCode: http.StatusBadRequest} - InvalidRange = &errorDescription{errorCode: "InvalidRange", errorMessage: "range is invalid", statusCode: http.StatusBadRequest} - UnauthorizedAccess = &errorDescription{errorCode: "UnauthorizedAccess", errorMessage: "UnauthorizedAccess", statusCode: http.StatusUnauthorized} - AccessDenied = &errorDescription{errorCode: "AccessDenied", errorMessage: "Access Denied", statusCode: http.StatusForbidden} - SignatureNotMatch = &errorDescription{errorCode: "SignatureDoesNotMatch", errorMessage: "SignatureDoesNotMatch", statusCode: http.StatusForbidden} - NoSuchKey = &errorDescription{errorCode: "NoSuchKey", errorMessage: "The specified key does not exist.", statusCode: http.StatusNotFound} - ObjectTxNotFound = &errorDescription{errorCode: "ObjectTxNotFound", errorMessage: "The specified object tx does not exist.", statusCode: http.StatusNotFound} - BucketAlreadyExists = &errorDescription{errorCode: "CreateBucketFailed", errorMessage: "Duplicate bucket name.", statusCode: http.StatusConflict} - ObjectAlreadyExists = &errorDescription{errorCode: "PutObjectFailed", errorMessage: "Duplicate object name.", statusCode: http.StatusConflict} + InvalidHeader = &errorDescription{errorCode: "InvalidHeader", errorMessage: "The headers maybe is invalid.", statusCode: http.StatusBadRequest} + InvalidBucketName = &errorDescription{errorCode: "InvalidBucketName", errorMessage: "The specified bucket is not valid.", statusCode: http.StatusBadRequest} + InvalidKey = &errorDescription{errorCode: "InvalidKey", errorMessage: "Object key is Illegal", statusCode: http.StatusBadRequest} + InvalidPayload = &errorDescription{errorCode: "InvalidPayload", errorMessage: "payload is empty", statusCode: http.StatusBadRequest} + InvalidObjectState = &errorDescription{errorCode: "InvalidObjectState", errorMessage: "object state is invalid", statusCode: http.StatusBadRequest} + InvalidRange = &errorDescription{errorCode: "InvalidRange", errorMessage: "range is invalid", statusCode: http.StatusBadRequest} + SignatureNotMatch = &errorDescription{errorCode: "SignatureDoesNotMatch", errorMessage: "SignatureDoesNotMatch", statusCode: http.StatusForbidden} + AccessDenied = &errorDescription{errorCode: "AccessDenied", errorMessage: "Access Denied", statusCode: http.StatusForbidden} + NoSuchKey = &errorDescription{errorCode: "NoSuchKey", errorMessage: "The specified key does not exist.", statusCode: http.StatusNotFound} + NoSuchBucket = &errorDescription{errorCode: "NoSuchBucket", errorMessage: "The specified bucket does not exist.", statusCode: http.StatusNotFound} // 5xx InternalError = &errorDescription{errorCode: "InternalError", errorMessage: "Internal Server Error", statusCode: http.StatusInternalServerError} NotImplementedError = &errorDescription{errorCode: "NotImplementedError", errorMessage: "Not Implemented Error", statusCode: http.StatusNotImplemented} @@ -103,3 +101,20 @@ func generateContentRangeHeader(w http.ResponseWriter, start int64, end int64) { w.Header().Set(model.ContentRangeHeader, "bytes "+util.Uint64ToString(uint64(start))+"-"+util.Uint64ToString(uint64(end))) } } + +func makeErrorDescription(err error) *errorDescription { + switch err { + case merrors.ErrNoSuchObject: + return NoSuchKey + case merrors.ErrNoSuchBucket: + return NoSuchBucket + case merrors.ErrAuthorizationFormat, merrors.ErrRequestConsistent, merrors.ErrSignatureConsistent, merrors.ErrUnsupportedSignType: + return SignatureNotMatch + case merrors.ErrNoPermission, merrors.ErrCheckPaymentAccountActive, merrors.ErrCheckQuotaEnough: + return AccessDenied + case merrors.ErrCheckObjectCreated, merrors.ErrCheckObjectSealed: + return InvalidObjectState + default: + return &errorDescription{errorCode: "InternalError", errorMessage: err.Error(), statusCode: http.StatusInternalServerError} + } +} diff --git a/service/gateway/sync_piece_handler.go b/service/gateway/sync_piece_handler.go index 98c1f5583..0606810bd 100644 --- a/service/gateway/sync_piece_handler.go +++ b/service/gateway/sync_piece_handler.go @@ -20,11 +20,11 @@ func (g *Gateway) syncPieceHandler(w http.ResponseWriter, r *http.Request) { errDescription *errorDescription reqContext *requestContext objectInfo = types.ObjectInfo{} - replicateIdx uint32 + replicaIdx uint32 segmentSize uint64 size int readN int - buf = make([]byte, model.StreamBufSize) + buf = make([]byte, model.DefaultStreamBufSize) integrityHash []byte integrityHashSignature []byte ) @@ -58,8 +58,8 @@ func (g *Gateway) syncPieceHandler(w http.ResponseWriter, r *http.Request) { errDescription = InvalidHeader return } - if replicateIdx, err = util.StringToUint32(r.Header.Get(model.GnfdReplicateIdxHeader)); err != nil { - log.Errorw("failed to parse replicate_idx header", "replicate_idx", r.Header.Get(model.GnfdReplicateIdxHeader)) + if replicaIdx, err = util.StringToUint32(r.Header.Get(model.GnfdReplicaIdxHeader)); err != nil { + log.Errorw("failed to parse replica_idx header", "replica_idx", r.Header.Get(model.GnfdReplicaIdxHeader)) errDescription = InvalidHeader return } @@ -84,10 +84,10 @@ func (g *Gateway) syncPieceHandler(w http.ResponseWriter, r *http.Request) { } if readN > 0 { if err = stream.Send(&syncertypes.SyncObjectRequest{ - ObjectInfo: &objectInfo, - ReplicateIdx: replicateIdx, - SegmentSize: segmentSize, - ReplicateData: buf[:readN], + ObjectInfo: &objectInfo, + ReplicaIdx: replicaIdx, + SegmentSize: segmentSize, + ReplicaData: buf[:readN], }); err != nil { log.Errorw("failed to send stream", "error", err) errDescription = InternalError diff --git a/service/syncer/syncer_service.go b/service/syncer/syncer_service.go index 22b82e6ab..e8e188f6d 100644 --- a/service/syncer/syncer_service.go +++ b/service/syncer/syncer_service.go @@ -73,7 +73,7 @@ func (syncer *Syncer) SyncObject(stream types.SyncerService_SyncObjectServer) (e if init { pstream.InitAsyncPayloadStream( req.GetObjectInfo().Id.Uint64(), - req.GetReplicateIdx(), + req.GetReplicaIdx(), req.GetSegmentSize(), req.GetObjectInfo().GetRedundancyType()) integrityMeta.ObjectID = req.GetObjectInfo().Id.Uint64() @@ -82,7 +82,7 @@ func (syncer *Syncer) SyncObject(stream types.SyncerService_SyncObjectServer) (e init = false } - pstream.StreamWrite(req.GetReplicateData()) + pstream.StreamWrite(req.GetReplicaData()) } }() diff --git a/service/uploader/client/uploader_client.go b/service/uploader/client/uploader_client.go index bceafabdb..b2561d8e3 100644 --- a/service/uploader/client/uploader_client.go +++ b/service/uploader/client/uploader_client.go @@ -36,18 +36,18 @@ func (client *UploaderClient) Close() error { return client.conn.Close() } -// QueryUploadingObject query an uploading object info with object id -func (client *UploaderClient) QueryUploadingObject(ctx context.Context, objectID uint64, opts ...grpc.CallOption) ( +// QueryPuttingObject query a putting object info with object id +func (client *UploaderClient) QueryPuttingObject(ctx context.Context, objectID uint64, opts ...grpc.CallOption) ( *servicetypes.SegmentInfo, error) { - resp, err := client.uploader.QueryUploadingObject(ctx, - &types.QueryUploadingObjectRequest{ObjectId: objectID}, opts...) + resp, err := client.uploader.QueryPuttingObject(ctx, + &types.QueryPuttingObjectRequest{ObjectId: objectID}, opts...) if err != nil { return nil, err } return resp.GetSegmentInfo(), nil } -// UploadObject return grpc stream client, and be used to upload object payload. -func (client *UploaderClient) UploadObject(ctx context.Context, opts ...grpc.CallOption) (types.UploaderService_UploadObjectClient, error) { - return client.uploader.UploadObject(ctx, opts...) +// PutObject return grpc stream client, and be used to upload object payload. +func (client *UploaderClient) PutObject(ctx context.Context, opts ...grpc.CallOption) (types.UploaderService_PutObjectClient, error) { + return client.uploader.PutObject(ctx, opts...) } diff --git a/service/uploader/uploader_service.go b/service/uploader/uploader_service.go index 9c9b20ca3..c232ec386 100644 --- a/service/uploader/uploader_service.go +++ b/service/uploader/uploader_service.go @@ -18,17 +18,17 @@ import ( var _ types.UploaderServiceServer = &Uploader{} -// UploadObject upload an object payload data with object info. -func (uploader *Uploader) UploadObject(stream types.UploaderService_UploadObjectServer) (err error) { +// PutObject upload an object payload data with object info. +func (uploader *Uploader) PutObject(stream types.UploaderService_PutObjectServer) (err error) { var ( - resp types.UploadObjectResponse + resp types.PutObjectResponse pstream = payloadstream.NewAsyncPayloadStream() traceInfo = &servicetypes.SegmentInfo{} checksum [][]byte integrityMeta = &sqldb.IntegrityMeta{} errCh = make(chan error, 10) ) - defer func(resp *types.UploadObjectResponse, err error) { + defer func(resp *types.PutObjectResponse, err error) { if err != nil { log.Errorw("failed to replicate payload", "err", err) uploader.spDB.UpdateJobState(traceInfo.GetObjectInfo().Id.Uint64(), @@ -135,12 +135,12 @@ func (uploader *Uploader) UploadObject(stream types.UploaderService_UploadObject } } -// QueryUploadingObject query an uploading object with object id from cache -func (uploader *Uploader) QueryUploadingObject(ctx context.Context, req *types.QueryUploadingObjectRequest) ( - resp *types.QueryUploadingObjectResponse, err error) { +// QueryPuttingObject query an uploading object with object id from cache +func (uploader *Uploader) QueryPuttingObject(ctx context.Context, req *types.QueryPuttingObjectRequest) ( + resp *types.QueryPuttingObjectResponse, err error) { ctx = log.Context(ctx, req) objectID := req.GetObjectId() - log.CtxDebugw(ctx, "query uploading object", "objectID", objectID) + log.CtxDebugw(ctx, "query putting object", "objectID", objectID) val, ok := uploader.cache.Get(objectID) if !ok { err = merrors.ErrCacheMiss diff --git a/store/piecestore/piece/piece_store.go b/store/piecestore/piece/piece_store.go index d30cafd00..921b40be9 100644 --- a/store/piecestore/piece/piece_store.go +++ b/store/piecestore/piece/piece_store.go @@ -88,7 +88,7 @@ func createStorage(cfg storage.PieceStoreConfig) (storage.ObjectStorage, error) func checkBucket(ctx context.Context, store storage.ObjectStorage) error { if err := store.HeadBucket(ctx); err != nil { log.Errorw("failed to head bucket", "error", err) - if errors.Is(err, merrors.ErrNotExistBucket) { + if errors.Is(err, merrors.ErrNoSuchBucket) { if err2 := store.CreateBucket(ctx); err2 != nil { return fmt.Errorf("failed to create bucket in %s: %s, previous err: %s", store, err2, err) } diff --git a/store/piecestore/storage/disk_file.go b/store/piecestore/storage/disk_file.go index 6410c21e2..0f1d2bafb 100644 --- a/store/piecestore/storage/disk_file.go +++ b/store/piecestore/storage/disk_file.go @@ -144,7 +144,7 @@ func (d *diskFileStore) DeleteObject(ctx context.Context, key string) error { func (d *diskFileStore) HeadBucket(ctx context.Context) error { if _, err := os.Stat(d.root); err != nil { if os.IsNotExist(err) { - return errors.ErrNotExistBucket + return errors.ErrNoSuchBucket } return err } diff --git a/store/piecestore/storage/memory.go b/store/piecestore/storage/memory.go index f4c995e6d..eea6898bf 100644 --- a/store/piecestore/storage/memory.go +++ b/store/piecestore/storage/memory.go @@ -46,7 +46,7 @@ func (m *memoryStore) GetObject(ctx context.Context, key string, offset, limit i } d, ok := m.objects[key] if !ok { - return nil, errors.ErrNotExitObject + return nil, errors.ErrNoSuchObject } if offset > int64(len(d.data)) { diff --git a/store/piecestore/storage/memory_test.go b/store/piecestore/storage/memory_test.go index 11ea597d2..1d6cfdb8c 100644 --- a/store/piecestore/storage/memory_test.go +++ b/store/piecestore/storage/memory_test.go @@ -67,7 +67,7 @@ func TestMemory_GetError(t *testing.T) { { name: "memory_get_error_test2", key: mockKey, - wantedErr: errors.ErrNotExitObject, + wantedErr: errors.ErrNoSuchObject, }, } for _, tt := range cases { diff --git a/store/piecestore/storage/s3.go b/store/piecestore/storage/s3.go index 77aada4cc..e455a40e6 100644 --- a/store/piecestore/storage/s3.go +++ b/store/piecestore/storage/s3.go @@ -148,7 +148,7 @@ func (s *s3Store) HeadBucket(ctx context.Context) error { log.Errorw("S3 failed to head bucket", "error", err) if reqErr, ok := err.(awserr.RequestFailure); ok { if reqErr.StatusCode() == http.StatusNotFound { - return merrors.ErrNotExistBucket + return merrors.ErrNoSuchBucket } } return err diff --git a/store/sqldb/entity.go b/store/sqldb/entity.go index 7acd24062..b0dbde8ff 100644 --- a/store/sqldb/entity.go +++ b/store/sqldb/entity.go @@ -70,5 +70,5 @@ func TimeUnixToTime(timeUnix int64) time.Time { // TimeToYearMonth convent time.Time to YYYY-MM string func TimeToYearMonth(t time.Time) string { - return t.Format("2006-01-02 15:04:05")[0:6] + return t.Format("2006-01-02 15:04:05")[0:7] } diff --git a/store/sqldb/sp.go b/store/sqldb/sp.go index decb94d63..2d5f8385a 100644 --- a/store/sqldb/sp.go +++ b/store/sqldb/sp.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "cosmossdk.io/math" + sdkmath "cosmossdk.io/math" sptypes "github.com/bnb-chain/greenfield/x/sp/types" storagetypes "github.com/bnb-chain/greenfield/x/storage/types" "gorm.io/gorm" @@ -41,17 +41,13 @@ func (s *SpDBImpl) UpdateAllSp(spList []*sptypes.StorageProvider) error { // insertNewRecordInSpInfoTable insert a new record in sp info table func (s *SpDBImpl) insertNewRecordInSpInfoTable(sp *sptypes.StorageProvider) error { - var totalDeposit int64 - if !sp.TotalDeposit.IsNil() && sp.TotalDeposit.IsInt64() { - totalDeposit = sp.TotalDeposit.Int64() - } insertRecord := &SpInfoTable{ OperatorAddress: sp.GetOperatorAddress(), IsOwn: false, FundingAddress: sp.GetFundingAddress(), SealAddress: sp.GetSealAddress(), ApprovalAddress: sp.GetApprovalAddress(), - TotalDeposit: totalDeposit, + TotalDeposit: sp.GetTotalDeposit().String(), Status: int32(sp.Status), Endpoint: sp.GetEndpoint(), Moniker: sp.GetDescription().Moniker, @@ -87,12 +83,16 @@ func (s *SpDBImpl) FetchAllSp(status ...sptypes.Status) ([]*sptypes.StorageProvi } records := []*sptypes.StorageProvider{} for _, value := range queryReturn { + totalDeposit, ok := sdkmath.NewIntFromString(value.TotalDeposit) + if !ok { + return records, fmt.Errorf("failed to parse int") + } records = append(records, &sptypes.StorageProvider{ OperatorAddress: value.OperatorAddress, FundingAddress: value.FundingAddress, SealAddress: value.SealAddress, ApprovalAddress: value.ApprovalAddress, - TotalDeposit: math.NewInt(value.TotalDeposit), + TotalDeposit: totalDeposit, Status: sptypes.Status(value.Status), Endpoint: value.Endpoint, Description: sptypes.Description{ @@ -132,12 +132,16 @@ func (s *SpDBImpl) FetchAllSpWithoutOwnSp(status ...sptypes.Status) ([]*sptypes. records := []*sptypes.StorageProvider{} for _, value := range queryReturn { + totalDeposit, ok := sdkmath.NewIntFromString(value.TotalDeposit) + if !ok { + return records, fmt.Errorf("failed to parse int") + } records = append(records, &sptypes.StorageProvider{ OperatorAddress: value.OperatorAddress, FundingAddress: value.FundingAddress, SealAddress: value.SealAddress, ApprovalAddress: value.ApprovalAddress, - TotalDeposit: math.NewInt(value.TotalDeposit), + TotalDeposit: totalDeposit, Status: sptypes.Status(value.Status), Endpoint: value.Endpoint, Description: sptypes.Description{ @@ -163,12 +167,16 @@ func (s *SpDBImpl) GetSpByAddress(address string, addressType SpAddressType) (*s if result.Error != nil { return nil, fmt.Errorf("failed to query sp info table: %s", result.Error) } + totalDeposit, ok := sdkmath.NewIntFromString(queryReturn.TotalDeposit) + if !ok { + return nil, fmt.Errorf("failed to parse int") + } return &sptypes.StorageProvider{ OperatorAddress: queryReturn.OperatorAddress, FundingAddress: queryReturn.FundingAddress, SealAddress: queryReturn.SealAddress, ApprovalAddress: queryReturn.ApprovalAddress, - TotalDeposit: math.NewInt(queryReturn.TotalDeposit), + TotalDeposit: totalDeposit, Status: sptypes.Status(queryReturn.Status), Endpoint: queryReturn.Endpoint, Description: sptypes.Description{ @@ -206,12 +214,16 @@ func (s *SpDBImpl) GetSpByEndpoint(endpoint string) (*sptypes.StorageProvider, e if result.Error != nil { return nil, fmt.Errorf("failed to query sp info table: %s", result.Error) } + totalDeposit, ok := sdkmath.NewIntFromString(queryReturn.TotalDeposit) + if !ok { + return nil, fmt.Errorf("failed to parse int") + } return &sptypes.StorageProvider{ OperatorAddress: queryReturn.OperatorAddress, FundingAddress: queryReturn.FundingAddress, SealAddress: queryReturn.SealAddress, ApprovalAddress: queryReturn.ApprovalAddress, - TotalDeposit: math.NewInt(queryReturn.TotalDeposit), + TotalDeposit: totalDeposit, Status: sptypes.Status(queryReturn.Status), Endpoint: queryReturn.Endpoint, Description: sptypes.Description{ @@ -231,12 +243,16 @@ func (s *SpDBImpl) GetOwnSpInfo() (*sptypes.StorageProvider, error) { if result.Error != nil { return nil, fmt.Errorf("failed to query own sp record in sp info table: %s", result.Error) } + totalDeposit, ok := sdkmath.NewIntFromString(queryReturn.TotalDeposit) + if !ok { + return nil, fmt.Errorf("failed to parse int") + } return &sptypes.StorageProvider{ OperatorAddress: queryReturn.OperatorAddress, FundingAddress: queryReturn.FundingAddress, SealAddress: queryReturn.SealAddress, ApprovalAddress: queryReturn.ApprovalAddress, - TotalDeposit: math.NewInt(queryReturn.TotalDeposit), + TotalDeposit: totalDeposit, Status: sptypes.Status(queryReturn.Status), Endpoint: queryReturn.Endpoint, Description: sptypes.Description{ @@ -262,7 +278,7 @@ func (s *SpDBImpl) SetOwnSpInfo(sp *sptypes.StorageProvider) error { FundingAddress: sp.GetFundingAddress(), SealAddress: sp.GetSealAddress(), ApprovalAddress: sp.GetApprovalAddress(), - TotalDeposit: sp.TotalDeposit.Int64(), + TotalDeposit: sp.GetTotalDeposit().String(), Status: int32(sp.GetStatus()), Endpoint: sp.GetEndpoint(), Moniker: sp.GetDescription().Moniker, @@ -281,7 +297,7 @@ func (s *SpDBImpl) SetOwnSpInfo(sp *sptypes.StorageProvider) error { } else { // if there is a record in SPInfoTable, update record result := s.db.Model(&SpInfoTable{}).Where("is_own = true").Updates(insertRecord) - if result.Error != nil || result.RowsAffected != 1 { + if result.Error != nil { return fmt.Errorf("failed to update own sp record in sp info table: %s", result.Error) } return nil diff --git a/store/sqldb/sp_schema.go b/store/sqldb/sp_schema.go index d1edfa497..6377d4b4e 100644 --- a/store/sqldb/sp_schema.go +++ b/store/sqldb/sp_schema.go @@ -7,7 +7,7 @@ type SpInfoTable struct { FundingAddress string SealAddress string ApprovalAddress string - TotalDeposit int64 + TotalDeposit string Status int32 Endpoint string Moniker string diff --git a/store/sqldb/traffic.go b/store/sqldb/traffic.go index 654608008..f8fa388c8 100644 --- a/store/sqldb/traffic.go +++ b/store/sqldb/traffic.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/bnb-chain/greenfield-storage-provider/model/errors" "gorm.io/gorm" ) @@ -18,7 +19,7 @@ func (s *SpDBImpl) CheckQuotaAndAddReadRecord(record *ReadRecord, quota *BucketQ // insert, if not existed insertBucketTraffic := &BucketTrafficTable{ BucketID: record.BucketID, - YearMonth: yearMonth, + Month: yearMonth, BucketName: record.BucketName, ReadCostSize: 0, ReadQuotaSize: quota.ReadQuotaSize, @@ -28,16 +29,18 @@ func (s *SpDBImpl) CheckQuotaAndAddReadRecord(record *ReadRecord, quota *BucketQ if result.Error != nil || result.RowsAffected != 1 { return fmt.Errorf("failed to insert bucket traffic table: %s", result.Error) } - bucketTraffic.BucketID = insertBucketTraffic.BucketID - bucketTraffic.YearMonth = insertBucketTraffic.YearMonth - bucketTraffic.BucketName = insertBucketTraffic.BucketName - bucketTraffic.ReadCostSize = insertBucketTraffic.ReadCostSize - bucketTraffic.ReadQuotaSize = insertBucketTraffic.ReadQuotaSize + bucketTraffic = &BucketTraffic{ + BucketID: insertBucketTraffic.BucketID, + YearMonth: insertBucketTraffic.Month, + BucketName: insertBucketTraffic.BucketName, + ReadCostSize: insertBucketTraffic.ReadCostSize, + ReadQuotaSize: insertBucketTraffic.ReadQuotaSize, + } } if bucketTraffic.ReadQuotaSize != quota.ReadQuotaSize { // update if chain quota has changed result := s.db.Model(&BucketTrafficTable{}). - Where("bucket_id = ? and year_month = ?", bucketTraffic.BucketID, bucketTraffic.YearMonth). + Where("bucket_id = ? and month = ?", bucketTraffic.BucketID, bucketTraffic.YearMonth). Updates(BucketTrafficTable{ ReadQuotaSize: quota.ReadQuotaSize, ModifiedTime: time.Now(), @@ -49,13 +52,13 @@ func (s *SpDBImpl) CheckQuotaAndAddReadRecord(record *ReadRecord, quota *BucketQ } // check quota - if bucketTraffic.ReadCostSize+record.ReadSize > bucketTraffic.ReadQuotaSize { - return fmt.Errorf("exceed read quota size") + if bucketTraffic.ReadCostSize+record.ReadSize > quota.ReadQuotaSize { + return errors.ErrCheckQuotaEnough } // update bucket traffic result := s.db.Model(&BucketTrafficTable{}). - Where("bucket_id = ? and year_month = ?", bucketTraffic.BucketID, bucketTraffic.YearMonth). + Where("bucket_id = ? and month = ?", bucketTraffic.BucketID, bucketTraffic.YearMonth). Updates(BucketTrafficTable{ ReadCostSize: bucketTraffic.ReadCostSize + record.ReadSize, ModifiedTime: time.Now(), @@ -88,17 +91,17 @@ func (s *SpDBImpl) GetBucketTraffic(bucketID uint64, yearMonth string) (*BucketT queryReturn BucketTrafficTable ) - result = s.db.First(&queryReturn, "bucket_id = ? and year_month = ?", bucketID, yearMonth) + result = s.db.Where("bucket_id = ? and month = ?", bucketID, yearMonth).First(&queryReturn) if result.Error == gorm.ErrRecordNotFound { // not found return nil, nil } if result.Error != nil { - return nil, fmt.Errorf("faile to query bucket traffic table: %s", result.Error) + return nil, fmt.Errorf("failed to query bucket traffic table: %s", result.Error) } return &BucketTraffic{ BucketID: queryReturn.BucketID, - YearMonth: queryReturn.YearMonth, + YearMonth: queryReturn.Month, BucketName: queryReturn.BucketName, ReadCostSize: queryReturn.ReadCostSize, ReadQuotaSize: queryReturn.ReadQuotaSize, diff --git a/store/sqldb/traffic_schema.go b/store/sqldb/traffic_schema.go index 6b296068e..c6943b00e 100644 --- a/store/sqldb/traffic_schema.go +++ b/store/sqldb/traffic_schema.go @@ -6,8 +6,8 @@ import ( // BucketTrafficTable table schema type BucketTrafficTable struct { - BucketID uint64 `gorm:"primary_key"` - YearMonth string `gorm:"primary_key"` + BucketID uint64 `gorm:"primary_key"` + Month string `gorm:"primary_key"` BucketName string ReadCostSize int64