Skip to content

Commit

Permalink
feat(backend): Renaming Aio and Null to Volume
Browse files Browse the repository at this point in the history
This brings more consistency to the naming of backend

See opiproject/opi-api#304

Signed-off-by: Boris Glimcher <[email protected]>
  • Loading branch information
glimchb committed Jul 26, 2023
1 parent 35dd5f8 commit 236c71d
Show file tree
Hide file tree
Showing 11 changed files with 197 additions and 209 deletions.
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -118,14 +118,14 @@ See services
```bash
$ grpc_cli ls opi-spdk-server:50051
grpc.reflection.v1alpha.ServerReflection
opi_api.storage.v1.AioControllerService
opi_api.storage.v1.AioVolumeService
opi_api.storage.v1.FrontendNvmeService
opi_api.storage.v1.FrontendVirtioBlkService
opi_api.storage.v1.FrontendVirtioScsiService
opi_api.storage.v1.MiddleendEncryptionService
opi_api.storage.v1.MiddleendQosVolumeService
opi_api.storage.v1.NvmeRemoteControllerService
opi_api.storage.v1.NullDebugService
opi_api.storage.v1.NullVolumeService
```

See commands
Expand Down
4 changes: 2 additions & 2 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,8 @@ func main() {
}

pb.RegisterNvmeRemoteControllerServiceServer(s, backendServer)
pb.RegisterNullDebugServiceServer(s, backendServer)
pb.RegisterAioControllerServiceServer(s, backendServer)
pb.RegisterNullVolumeServiceServer(s, backendServer)
pb.RegisterAioVolumeServiceServer(s, backendServer)
pb.RegisterMiddleendEncryptionServiceServer(s, middleendServer)
pb.RegisterMiddleendQosVolumeServiceServer(s, middleendServer)

Expand Down
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/digitalocean/go-qemu v0.0.0-20230711162256-2e3d0186973e
github.com/google/uuid v1.3.0
github.com/opiproject/gospdk v0.0.0-20230721162442-5187c4c6663b
github.com/opiproject/opi-api v0.0.0-20230721161716-ea8314a63ccb
github.com/opiproject/opi-api v0.0.0-20230726170919-691a90a13429
go.einride.tech/aip v0.60.0
google.golang.org/grpc v1.56.2
google.golang.org/protobuf v1.31.0
Expand All @@ -18,7 +18,7 @@ require (
golang.org/x/net v0.12.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e // indirect
)
28 changes: 8 additions & 20 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e h1:SCnqm8SjSa0QqRxXbo5YY//S+OryeJioe17nK+iDZpg=
github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e/go.mod h1:o129ljs6alsIQTc8d6eweihqpmmrbxZ2g1jhgjhPykI=
github.com/digitalocean/go-qemu v0.0.0-20221209210016-f035778c97f7 h1:3OVJAbR131SnAXao7c9w8bFlAGH0oa29DCwsa88MJGk=
github.com/digitalocean/go-qemu v0.0.0-20221209210016-f035778c97f7/go.mod h1:K4+o74YGNjOb9N6yyG+LPj1NjHtk+Qz0IYQPvirbaLs=
github.com/digitalocean/go-qemu v0.0.0-20230711162256-2e3d0186973e h1:x5PInTuXLddHWHlePCNAcM8QtUfOGx44f3UmYPMtDcI=
github.com/digitalocean/go-qemu v0.0.0-20230711162256-2e3d0186973e/go.mod h1:K4+o74YGNjOb9N6yyG+LPj1NjHtk+Qz0IYQPvirbaLs=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
Expand All @@ -13,20 +11,10 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/opiproject/gospdk v0.0.0-20230706153333-46d1efd3dfde h1:F34T5Kq7GzFSlnycskJ/LXfBDwhsN061ADzhARHO+Lg=
github.com/opiproject/gospdk v0.0.0-20230706153333-46d1efd3dfde/go.mod h1:UzRy421kjqvDVi1awOCLaFpyMLAGKMR3G5tXgqLsiq8=
github.com/opiproject/gospdk v0.0.0-20230714152149-de73bd1ee87d h1:H+E4ITds+AW3j7PHLbhAz3C+OPGJGREJUdmTo9YOviE=
github.com/opiproject/gospdk v0.0.0-20230714152149-de73bd1ee87d/go.mod h1:RqA5Ix7+x0Is8UckJE+6Ji5kCfk6yujWWaTMoRlZv3w=
github.com/opiproject/gospdk v0.0.0-20230721162442-5187c4c6663b h1:MYk4lS7uUpJwT89WJgjqg9RCXywslytpBDmZcWy626I=
github.com/opiproject/gospdk v0.0.0-20230721162442-5187c4c6663b/go.mod h1:RqA5Ix7+x0Is8UckJE+6Ji5kCfk6yujWWaTMoRlZv3w=
github.com/opiproject/opi-api v0.0.0-20230712141241-3ccebf87270b h1:/t1mBhxyV8Obu2zeVhAjyXaLfy4dwCrS1A8/MJ71KFA=
github.com/opiproject/opi-api v0.0.0-20230712141241-3ccebf87270b/go.mod h1:92pv4ulvvPMuxCJ9ND3aYbmBfEMLx0VCjpkiR7ZTqPY=
github.com/opiproject/opi-api v0.0.0-20230713203751-f1f72eaaee0e h1:dFlwXYeXuRPKe5w40eDYMpc3+1zCQDXS+9W/5LWCnbU=
github.com/opiproject/opi-api v0.0.0-20230713203751-f1f72eaaee0e/go.mod h1:92pv4ulvvPMuxCJ9ND3aYbmBfEMLx0VCjpkiR7ZTqPY=
github.com/opiproject/opi-api v0.0.0-20230717064243-520b62d2f155 h1:I35TX9YBVrl6PlYe1Eima79mA6VS0qohETDUoB6W+Pk=
github.com/opiproject/opi-api v0.0.0-20230717064243-520b62d2f155/go.mod h1:92pv4ulvvPMuxCJ9ND3aYbmBfEMLx0VCjpkiR7ZTqPY=
github.com/opiproject/opi-api v0.0.0-20230721161716-ea8314a63ccb h1:N8wK6C6A7Y4fr9SECSvTTUMhcD2xZ6t2W+/NO9vy768=
github.com/opiproject/opi-api v0.0.0-20230721161716-ea8314a63ccb/go.mod h1:92pv4ulvvPMuxCJ9ND3aYbmBfEMLx0VCjpkiR7ZTqPY=
github.com/opiproject/opi-api v0.0.0-20230726170919-691a90a13429 h1:bvctyHEk77amCzqlP2Q40C7uTzgYCX/IEdpmAkvFw0Y=
github.com/opiproject/opi-api v0.0.0-20230726170919-691a90a13429/go.mod h1:92pv4ulvvPMuxCJ9ND3aYbmBfEMLx0VCjpkiR7ZTqPY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
Expand Down Expand Up @@ -64,12 +52,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g=
google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0=
google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw=
google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e h1:xIXmWJ303kJCuogpj0bHq+dcjcZHU+XFyc1I0Yl9cRg=
google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:0ggbjUrZYpy1q+ANUS30SEoGZ53cdfwtbuG7Ptgy108=
google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw=
google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e h1:S83+ibolgyZ0bqz7KEsUOPErxcv4VzlszxY+31OfB/E=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
Expand Down
96 changes: 48 additions & 48 deletions pkg/backend/aio.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,43 +26,43 @@ import (
"google.golang.org/protobuf/types/known/emptypb"
)

func sortAioControllers(controllers []*pb.AioController) {
func sortAioVolumes(controllers []*pb.AioVolume) {
sort.Slice(controllers, func(i int, j int) bool {
return controllers[i].Name < controllers[j].Name
})
}

// CreateAioController creates an Aio controller
func (s *Server) CreateAioController(_ context.Context, in *pb.CreateAioControllerRequest) (*pb.AioController, error) {
log.Printf("CreateAioController: Received from client: %v", in)
// CreateAioVolume creates an Aio controller
func (s *Server) CreateAioVolume(_ context.Context, in *pb.CreateAioVolumeRequest) (*pb.AioVolume, error) {
log.Printf("CreateAioVolume: Received from client: %v", in)
// check required fields
if err := fieldbehavior.ValidateRequiredFields(in); err != nil {
log.Printf("error: %v", err)
return nil, err
}
// see https://google.aip.dev/133#user-specified-ids
resourceID := resourceid.NewSystemGenerated()
if in.AioControllerId != "" {
err := resourceid.ValidateUserSettable(in.AioControllerId)
if in.AioVolumeId != "" {
err := resourceid.ValidateUserSettable(in.AioVolumeId)
if err != nil {
log.Printf("error: %v", err)
return nil, err
}
log.Printf("client provided the ID of a resource %v, ignoring the name field %v", in.AioControllerId, in.AioController.Name)
resourceID = in.AioControllerId
log.Printf("client provided the ID of a resource %v, ignoring the name field %v", in.AioVolumeId, in.AioVolume.Name)
resourceID = in.AioVolumeId
}
in.AioController.Name = server.ResourceIDToVolumeName(resourceID)
in.AioVolume.Name = server.ResourceIDToVolumeName(resourceID)
// idempotent API when called with same key, should return same object
volume, ok := s.Volumes.AioVolumes[in.AioController.Name]
volume, ok := s.Volumes.AioVolumes[in.AioVolume.Name]
if ok {
log.Printf("Already existing AioController with id %v", in.AioController.Name)
log.Printf("Already existing AioVolume with id %v", in.AioVolume.Name)
return volume, nil
}
// not found, so create a new one
params := spdk.BdevAioCreateParams{
Name: resourceID,
BlockSize: 512,
Filename: in.AioController.Filename,
Filename: in.AioVolume.Filename,
}
var result spdk.BdevAioCreateResult
err := s.rpc.Call("bdev_aio_create", &params, &result)
Expand All @@ -76,15 +76,15 @@ func (s *Server) CreateAioController(_ context.Context, in *pb.CreateAioControll
log.Print(msg)
return nil, status.Errorf(codes.InvalidArgument, msg)
}
response := server.ProtoClone(in.AioController)
s.Volumes.AioVolumes[in.AioController.Name] = response
log.Printf("CreateAioController: Sending to client: %v", response)
response := server.ProtoClone(in.AioVolume)
s.Volumes.AioVolumes[in.AioVolume.Name] = response
log.Printf("CreateAioVolume: Sending to client: %v", response)
return response, nil
}

// DeleteAioController deletes an Aio controller
func (s *Server) DeleteAioController(_ context.Context, in *pb.DeleteAioControllerRequest) (*emptypb.Empty, error) {
log.Printf("DeleteAioController: Received from client: %v", in)
// DeleteAioVolume deletes an Aio controller
func (s *Server) DeleteAioVolume(_ context.Context, in *pb.DeleteAioVolumeRequest) (*emptypb.Empty, error) {
log.Printf("DeleteAioVolume: Received from client: %v", in)
// check required fields
if err := fieldbehavior.ValidateRequiredFields(in); err != nil {
log.Printf("error: %v", err)
Expand Down Expand Up @@ -125,28 +125,28 @@ func (s *Server) DeleteAioController(_ context.Context, in *pb.DeleteAioControll
return &emptypb.Empty{}, nil
}

// UpdateAioController updates an Aio controller
func (s *Server) UpdateAioController(_ context.Context, in *pb.UpdateAioControllerRequest) (*pb.AioController, error) {
log.Printf("UpdateAioController: Received from client: %v", in)
// UpdateAioVolume updates an Aio controller
func (s *Server) UpdateAioVolume(_ context.Context, in *pb.UpdateAioVolumeRequest) (*pb.AioVolume, error) {
log.Printf("UpdateAioVolume: Received from client: %v", in)
// check required fields
if err := fieldbehavior.ValidateRequiredFields(in); err != nil {
log.Printf("error: %v", err)
return nil, err
}
// Validate that a resource name conforms to the restrictions outlined in AIP-122.
if err := resourcename.Validate(in.AioController.Name); err != nil {
if err := resourcename.Validate(in.AioVolume.Name); err != nil {
log.Printf("error: %v", err)
return nil, err
}
// fetch object from the database
volume, ok := s.Volumes.AioVolumes[in.AioController.Name]
volume, ok := s.Volumes.AioVolumes[in.AioVolume.Name]
if !ok {
if in.AllowMissing {
log.Printf("Got AllowMissing, create a new resource, don't return error when resource not found")
params := spdk.BdevAioCreateParams{
Name: path.Base(in.AioController.Name),
Name: path.Base(in.AioVolume.Name),
BlockSize: 512,
Filename: in.AioController.Filename,
Filename: in.AioVolume.Filename,
}
var result spdk.BdevAioCreateResult
err := s.rpc.Call("bdev_aio_create", &params, &result)
Expand All @@ -160,18 +160,18 @@ func (s *Server) UpdateAioController(_ context.Context, in *pb.UpdateAioControll
log.Print(msg)
return nil, status.Errorf(codes.InvalidArgument, msg)
}
response := server.ProtoClone(in.AioController)
s.Volumes.AioVolumes[in.AioController.Name] = response
log.Printf("CreateAioController: Sending to client: %v", response)
response := server.ProtoClone(in.AioVolume)
s.Volumes.AioVolumes[in.AioVolume.Name] = response
log.Printf("CreateAioVolume: Sending to client: %v", response)
return response, nil
}
err := status.Errorf(codes.NotFound, "unable to find key %s", in.AioController.Name)
err := status.Errorf(codes.NotFound, "unable to find key %s", in.AioVolume.Name)
log.Printf("error: %v", err)
return nil, err
}
resourceID := path.Base(volume.Name)
// update_mask = 2
if err := fieldmask.Validate(in.UpdateMask, in.AioController); err != nil {
if err := fieldmask.Validate(in.UpdateMask, in.AioVolume); err != nil {
log.Printf("error: %v", err)
return nil, err
}
Expand All @@ -193,7 +193,7 @@ func (s *Server) UpdateAioController(_ context.Context, in *pb.UpdateAioControll
params2 := spdk.BdevAioCreateParams{
Name: resourceID,
BlockSize: 512,
Filename: in.AioController.Filename,
Filename: in.AioVolume.Filename,
}
var result2 spdk.BdevAioCreateResult
err2 := s.rpc.Call("bdev_aio_create", &params2, &result2)
Expand All @@ -207,14 +207,14 @@ func (s *Server) UpdateAioController(_ context.Context, in *pb.UpdateAioControll
log.Print(msg)
return nil, status.Errorf(codes.InvalidArgument, msg)
}
response := server.ProtoClone(in.AioController)
s.Volumes.AioVolumes[in.AioController.Name] = response
response := server.ProtoClone(in.AioVolume)
s.Volumes.AioVolumes[in.AioVolume.Name] = response
return response, nil
}

// ListAioControllers lists Aio controllers
func (s *Server) ListAioControllers(_ context.Context, in *pb.ListAioControllersRequest) (*pb.ListAioControllersResponse, error) {
log.Printf("ListAioControllers: Received from client: %v", in)
// ListAioVolumes lists Aio controllers
func (s *Server) ListAioVolumes(_ context.Context, in *pb.ListAioVolumesRequest) (*pb.ListAioVolumesResponse, error) {
log.Printf("ListAioVolumes: Received from client: %v", in)
// check required fields
if err := fieldbehavior.ValidateRequiredFields(in); err != nil {
log.Printf("error: %v", err)
Expand All @@ -240,18 +240,18 @@ func (s *Server) ListAioControllers(_ context.Context, in *pb.ListAioControllers
token = uuid.New().String()
s.Pagination[token] = offset + size
}
Blobarray := make([]*pb.AioController, len(result))
Blobarray := make([]*pb.AioVolume, len(result))
for i := range result {
r := &result[i]
Blobarray[i] = &pb.AioController{Name: r.Name, BlockSize: r.BlockSize, BlocksCount: r.NumBlocks}
Blobarray[i] = &pb.AioVolume{Name: r.Name, BlockSize: r.BlockSize, BlocksCount: r.NumBlocks}
}
sortAioControllers(Blobarray)
return &pb.ListAioControllersResponse{AioControllers: Blobarray, NextPageToken: token}, nil
sortAioVolumes(Blobarray)
return &pb.ListAioVolumesResponse{AioVolumes: Blobarray, NextPageToken: token}, nil
}

// GetAioController gets an Aio controller
func (s *Server) GetAioController(_ context.Context, in *pb.GetAioControllerRequest) (*pb.AioController, error) {
log.Printf("GetAioController: Received from client: %v", in)
// GetAioVolume gets an Aio controller
func (s *Server) GetAioVolume(_ context.Context, in *pb.GetAioVolumeRequest) (*pb.AioVolume, error) {
log.Printf("GetAioVolume: Received from client: %v", in)
// check required fields
if err := fieldbehavior.ValidateRequiredFields(in); err != nil {
log.Printf("error: %v", err)
Expand Down Expand Up @@ -285,12 +285,12 @@ func (s *Server) GetAioController(_ context.Context, in *pb.GetAioControllerRequ
log.Print(msg)
return nil, status.Errorf(codes.InvalidArgument, msg)
}
return &pb.AioController{Name: result[0].Name, BlockSize: result[0].BlockSize, BlocksCount: result[0].NumBlocks}, nil
return &pb.AioVolume{Name: result[0].Name, BlockSize: result[0].BlockSize, BlocksCount: result[0].NumBlocks}, nil
}

// AioControllerStats gets an Aio controller stats
func (s *Server) AioControllerStats(_ context.Context, in *pb.AioControllerStatsRequest) (*pb.AioControllerStatsResponse, error) {
log.Printf("AioControllerStats: Received from client: %v", in)
// AioVolumeStats gets an Aio controller stats
func (s *Server) AioVolumeStats(_ context.Context, in *pb.AioVolumeStatsRequest) (*pb.AioVolumeStatsResponse, error) {
log.Printf("AioVolumeStats: Received from client: %v", in)
// check required fields
if err := fieldbehavior.ValidateRequiredFields(in); err != nil {
log.Printf("error: %v", err)
Expand Down Expand Up @@ -325,7 +325,7 @@ func (s *Server) AioControllerStats(_ context.Context, in *pb.AioControllerStats
log.Print(msg)
return nil, status.Errorf(codes.InvalidArgument, msg)
}
return &pb.AioControllerStatsResponse{Stats: &pb.VolumeStats{
return &pb.AioVolumeStatsResponse{Stats: &pb.VolumeStats{
ReadBytesCount: int32(result.Bdevs[0].BytesRead),
ReadOpsCount: int32(result.Bdevs[0].NumReadOps),
WriteBytesCount: int32(result.Bdevs[0].BytesWritten),
Expand Down
Loading

0 comments on commit 236c71d

Please sign in to comment.