Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

K8SPXC-1542: Fix Azure binlog upload for large files #1848

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

K8SPXC-1542: Fix Azure binlog upload for large files #1848

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ee0c387
Fix Azure binlog upload for large files
dcaputo-harmoni Oct 16, 2024
d86af6e
Merge branch 'main' into GITHUB-ISSUE-1847-fix-azure-binlog-upload-fo…
hors Oct 22, 2024
fb6dd1c
Merge branch 'main' into GITHUB-ISSUE-1847-fix-azure-binlog-upload-fo…
hors Nov 26, 2024
7440c2e
Make block size and concurrency configurable for Azure storage
dcaputo-harmoni Dec 15, 2024
09b4902
Merge branch 'main' into GITHUB-ISSUE-1847-fix-azure-binlog-upload-fo…
hors Jan 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion cmd/pitr/collector/collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ type BackupAzure struct {
StorageClass string `env:"AZURE_STORAGE_CLASS"`
AccountName string `env:"AZURE_STORAGE_ACCOUNT,required"`
AccountKey string `env:"AZURE_ACCESS_KEY,required"`
BlockSize int64 `env:"AZURE_BLOCK_SIZE"`
Concurrency int `env:"AZURE_CONCURRENCY"`
}

const (
Expand Down Expand Up @@ -85,7 +87,7 @@ func New(ctx context.Context, c Config) (*Collector, error) {
if prefix != "" {
prefix += "/"
}
s, err = storage.NewAzure(c.BackupStorageAzure.AccountName, c.BackupStorageAzure.AccountKey, c.BackupStorageAzure.Endpoint, container, prefix)
s, err = storage.NewAzure(c.BackupStorageAzure.AccountName, c.BackupStorageAzure.AccountKey, c.BackupStorageAzure.Endpoint, container, prefix, c.BackupStorageAzure.BlockSize, c.BackupStorageAzure.Concurrency)
if err != nil {
return nil, errors.Wrap(err, "new azure storage")
}
Expand Down
8 changes: 6 additions & 2 deletions cmd/pitr/recoverer/recoverer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,11 @@ func (c Config) storages(ctx context.Context) (storage.Storage, storage.Storage,
case "azure":
var err error
container, prefix := getContainerAndPrefix(c.BinlogStorageAzure.ContainerPath)
binlogStorage, err = storage.NewAzure(c.BinlogStorageAzure.AccountName, c.BinlogStorageAzure.AccountKey, c.BinlogStorageAzure.Endpoint, container, prefix)
binlogStorage, err = storage.NewAzure(c.BinlogStorageAzure.AccountName, c.BinlogStorageAzure.AccountKey, c.BinlogStorageAzure.Endpoint, container, prefix, c.BinlogStorageAzure.BlockSize, c.BinlogStorageAzure.Concurrency)
if err != nil {
return nil, nil, errors.Wrap(err, "new azure storage")
}
defaultStorage, err = storage.NewAzure(c.BackupStorageAzure.AccountName, c.BackupStorageAzure.AccountKey, c.BackupStorageAzure.Endpoint, c.BackupStorageAzure.ContainerName, c.BackupStorageAzure.BackupDest+".sst_info/")
defaultStorage, err = storage.NewAzure(c.BackupStorageAzure.AccountName, c.BackupStorageAzure.AccountKey, c.BackupStorageAzure.Endpoint, c.BackupStorageAzure.ContainerName, c.BackupStorageAzure.BackupDest+".sst_info/", c.BackupStorageAzure.BlockSize, c.BackupStorageAzure.Concurrency)
if err != nil {
return nil, nil, errors.Wrap(err, "new azure storage")
}
Expand All @@ -105,6 +105,8 @@ type BackupAzure struct {
AccountName string `env:"AZURE_STORAGE_ACCOUNT,required"`
AccountKey string `env:"AZURE_ACCESS_KEY,required"`
BackupDest string `env:"BACKUP_PATH,required"`
BlockSize int64 `env:"AZURE_BLOCK_SIZE"`
Concurrency int `env:"AZURE_CONCURRENCY"`
}

type BinlogS3 struct {
Expand All @@ -121,6 +123,8 @@ type BinlogAzure struct {
StorageClass string `env:"BINLOG_AZURE_STORAGE_CLASS"`
AccountName string `env:"BINLOG_AZURE_STORAGE_ACCOUNT,required"`
AccountKey string `env:"BINLOG_AZURE_ACCESS_KEY,required"`
BlockSize int64 `env:"BINLOG_AZURE_BLOCK_SIZE"`
Concurrency int `env:"BINLOG_AZURE_CONCURRENCY"`
}

func (c *Config) Verify() {
Expand Down
5 changes: 5 additions & 0 deletions config/crd/bases/pxc.percona.com_perconaxtradbclusterbackups.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
completed:
format: date-time
Expand Down
10 changes: 10 additions & 0 deletions config/crd/bases/pxc.percona.com_perconaxtradbclusterrestores.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
completed:
format: date-time
Expand Down Expand Up @@ -233,6 +238,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
completed:
format: date-time
Expand Down
5 changes: 5 additions & 0 deletions config/crd/bases/pxc.percona.com_perconaxtradbclusters.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -599,6 +599,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
containerOptions:
properties:
Expand Down
20 changes: 20 additions & 0 deletions deploy/bundle.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
completed:
format: date-time
Expand Down Expand Up @@ -302,6 +307,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
completed:
format: date-time
Expand Down Expand Up @@ -476,6 +486,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
completed:
format: date-time
Expand Down Expand Up @@ -1504,6 +1519,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
containerOptions:
properties:
Expand Down
2 changes: 2 additions & 0 deletions deploy/cr.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -672,6 +672,8 @@ spec:
container: test
# endpointUrl: https://accountName.blob.core.windows.net
# storageClass: Hot
# blockSize: 4194304
# concurrency: 4
fs-pvc:
type: filesystem
# nodeSelector:
Expand Down
20 changes: 20 additions & 0 deletions deploy/crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
completed:
format: date-time
Expand Down Expand Up @@ -302,6 +307,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
completed:
format: date-time
Expand Down Expand Up @@ -476,6 +486,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
completed:
format: date-time
Expand Down Expand Up @@ -1504,6 +1519,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
containerOptions:
properties:
Expand Down
20 changes: 20 additions & 0 deletions deploy/cw-bundle.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
completed:
format: date-time
Expand Down Expand Up @@ -302,6 +307,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
completed:
format: date-time
Expand Down Expand Up @@ -476,6 +486,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
completed:
format: date-time
Expand Down Expand Up @@ -1504,6 +1519,11 @@ spec:
type: string
storageClass:
type: string
blockSize:
format: int64
type: integer
concurrency:
type: integer
type: object
containerOptions:
properties:
Expand Down
2 changes: 2 additions & 0 deletions pkg/apis/pxc/v1/pxc_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -725,6 +725,8 @@ type BackupStorageAzureSpec struct {
ContainerPath string `json:"container"`
Endpoint string `json:"endpointUrl"`
StorageClass string `json:"storageClass"`
BlockSize int64 `json:"blockSize"`
Concurrency int `json:"concurrency"`
}

const (
Expand Down
4 changes: 4 additions & 0 deletions pkg/pxc/backup/storage/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ func getAzureOptions(ctx context.Context, cl client.Client, backup *api.PerconaX
Endpoint: backup.Status.Azure.Endpoint,
Container: container,
Prefix: prefix,
BlockSize: backup.Status.Azure.BlockSize,
Concurrency: backup.Status.Azure.Concurrency,
}, nil
}

Expand Down Expand Up @@ -127,6 +129,8 @@ type AzureOptions struct {
Endpoint string
Container string
Prefix string
BlockSize int64
Concurrency int
}

func (o *AzureOptions) Type() api.BackupStorageType {
Expand Down
29 changes: 20 additions & 9 deletions pkg/pxc/backup/storage/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func NewClient(ctx context.Context, opts Options) (Storage, error) {
if !ok {
return nil, errors.New("invalid options type")
}
return NewAzure(opts.StorageAccount, opts.AccessKey, opts.Endpoint, opts.Container, opts.Prefix)
return NewAzure(opts.StorageAccount, opts.AccessKey, opts.Endpoint, opts.Container, opts.Prefix, opts.BlockSize, opts.Concurrency)
}
return nil, errors.New("invalid storage type")
}
Expand Down Expand Up @@ -188,12 +188,14 @@ func (s *S3) DeleteObject(ctx context.Context, objectName string) error {

// Azure is a type for working with Azure Blob storages
type Azure struct {
client *azblob.Client // azure client for work with storage
container string
prefix string
client *azblob.Client // azure client for work with storage
container string
prefix string
blockSize int64
concurrency int
}

func NewAzure(storageAccount, accessKey, endpoint, container, prefix string) (Storage, error) {
func NewAzure(storageAccount, accessKey, endpoint, container, prefix string, blockSize int64, concurrency int) (Storage, error) {
credential, err := azblob.NewSharedKeyCredential(storageAccount, accessKey)
if err != nil {
return nil, errors.Wrap(err, "new credentials")
Expand All @@ -207,9 +209,11 @@ func NewAzure(storageAccount, accessKey, endpoint, container, prefix string) (St
}

return &Azure{
client: cli,
container: container,
prefix: prefix,
client: cli,
container: container,
prefix: prefix,
blockSize: blockSize,
concurrency: concurrency,
}, nil
}

Expand All @@ -227,7 +231,14 @@ func (a *Azure) GetObject(ctx context.Context, name string) (io.ReadCloser, erro

func (a *Azure) PutObject(ctx context.Context, name string, data io.Reader, _ int64) error {
objPath := path.Join(a.prefix, name)
_, err := a.client.UploadStream(ctx, a.container, objPath, data, nil)
uploadOptions := azblob.UploadStreamOptions{}
if a.blockSize > 0 {
uploadOptions.BlockSize = a.blockSize
}
dcaputo-harmoni marked this conversation as resolved.
Show resolved Hide resolved
if a.concurrency > 0 {
uploadOptions.Concurrency = a.concurrency
}
_, err := a.client.UploadStream(ctx, a.container, objPath, data, &uploadOptions)
if err != nil {
return errors.Wrapf(err, "upload stream: %s", objPath)
}
Expand Down
Loading