-
Notifications
You must be signed in to change notification settings - Fork 362
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add StorageID to Repository entity #8502
Changes from 8 commits
c1d3587
4e51981
30b3849
02e4404
74e2202
222b126
af9476e
4e8e423
989a388
298bc0d
299c292
b81eff3
acb70e8
3dbd9d1
1ced59b
6d2d2e7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -446,8 +446,9 @@ func (c *Catalog) log(ctx context.Context) logging.Logger { | |
} | ||
|
||
// CreateRepository create a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with default branch name 'branch' | ||
func (c *Catalog) CreateRepository(ctx context.Context, repository string, storageNamespace string, branch string, readOnly bool) (*Repository, error) { | ||
func (c *Catalog) CreateRepository(ctx context.Context, repository string, storageID string, storageNamespace string, branch string, readOnly bool) (*Repository, error) { | ||
repositoryID := graveler.RepositoryID(repository) | ||
storageIdentifier := graveler.StorageID(storageID) | ||
storageNS := graveler.StorageNamespace(storageNamespace) | ||
branchID := graveler.BranchID(branch) | ||
if err := validator.Validate([]validator.ValidateArg{ | ||
|
@@ -456,12 +457,13 @@ func (c *Catalog) CreateRepository(ctx context.Context, repository string, stora | |
}); err != nil { | ||
return nil, err | ||
} | ||
repo, err := c.Store.CreateRepository(ctx, repositoryID, storageNS, branchID, readOnly) | ||
repo, err := c.Store.CreateRepository(ctx, repositoryID, storageIdentifier, storageNS, branchID, readOnly) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As mentioned in the previous comment and also discussed with @guy-har. |
||
if err != nil { | ||
return nil, err | ||
} | ||
catalogRepo := &Repository{ | ||
Name: repositoryID.String(), | ||
StorageID: storageIdentifier.String(), | ||
StorageNamespace: storageNS.String(), | ||
DefaultBranch: branchID.String(), | ||
CreationDate: repo.CreationDate, | ||
|
@@ -472,8 +474,9 @@ func (c *Catalog) CreateRepository(ctx context.Context, repository string, stora | |
|
||
// CreateBareRepository create a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with no initial branch or commit | ||
// defaultBranchID will point to a non-existent branch on creation, it is up to the caller to eventually create it. | ||
func (c *Catalog) CreateBareRepository(ctx context.Context, repository string, storageNamespace string, defaultBranchID string, readOnly bool) (*Repository, error) { | ||
func (c *Catalog) CreateBareRepository(ctx context.Context, repository string, storageID string, storageNamespace string, defaultBranchID string, readOnly bool) (*Repository, error) { | ||
repositoryID := graveler.RepositoryID(repository) | ||
storageIdentifier := graveler.StorageID(storageID) | ||
storageNS := graveler.StorageNamespace(storageNamespace) | ||
branchID := graveler.BranchID(defaultBranchID) | ||
if err := validator.Validate([]validator.ValidateArg{ | ||
|
@@ -482,12 +485,13 @@ func (c *Catalog) CreateBareRepository(ctx context.Context, repository string, s | |
}); err != nil { | ||
return nil, err | ||
} | ||
repo, err := c.Store.CreateBareRepository(ctx, repositoryID, storageNS, branchID, readOnly) | ||
repo, err := c.Store.CreateBareRepository(ctx, repositoryID, storageIdentifier, storageNS, branchID, readOnly) | ||
if err != nil { | ||
return nil, err | ||
} | ||
catalogRepo := &Repository{ | ||
Name: repositoryID.String(), | ||
StorageID: storageIdentifier.String(), | ||
StorageNamespace: storageNS.String(), | ||
DefaultBranch: branchID.String(), | ||
CreationDate: repo.CreationDate, | ||
|
@@ -516,6 +520,7 @@ func (c *Catalog) GetRepository(ctx context.Context, repository string) (*Reposi | |
|
||
catalogRepository := &Repository{ | ||
Name: repositoryID.String(), | ||
StorageID: repo.StorageID.String(), | ||
StorageNamespace: repo.StorageNamespace.String(), | ||
DefaultBranch: repo.DefaultBranchID.String(), | ||
CreationDate: repo.CreationDate, | ||
|
@@ -623,6 +628,7 @@ func (c *Catalog) ListRepositories(ctx context.Context, limit int, prefix, searc | |
} | ||
repos = append(repos, &Repository{ | ||
Name: record.RepositoryID.String(), | ||
StorageID: record.StorageID.String(), | ||
StorageNamespace: record.StorageNamespace.String(), | ||
DefaultBranch: record.DefaultBranchID.String(), | ||
CreationDate: record.CreationDate, | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -72,12 +72,12 @@ func TestCatalog_ListRepositories(t *testing.T) { | |
// prepare data tests | ||
now := time.Now() | ||
gravelerData := []*graveler.RepositoryRecord{ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please make sure to have one record with no StorageID (empty) as part of our tests There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Right. Added. |
||
{RepositoryID: "re", Repository: &graveler.Repository{StorageNamespace: "storage1", CreationDate: now, DefaultBranchID: "main1"}}, | ||
{RepositoryID: "repo1", Repository: &graveler.Repository{StorageNamespace: "storage2", CreationDate: now, DefaultBranchID: "main2"}}, | ||
{RepositoryID: "repo2", Repository: &graveler.Repository{StorageNamespace: "storage3", CreationDate: now, DefaultBranchID: "main3"}}, | ||
{RepositoryID: "repo22", Repository: &graveler.Repository{StorageNamespace: "storage4", CreationDate: now, DefaultBranchID: "main4"}}, | ||
{RepositoryID: "repo23", Repository: &graveler.Repository{StorageNamespace: "storage5", CreationDate: now, DefaultBranchID: "main5"}}, | ||
{RepositoryID: "repo3", Repository: &graveler.Repository{StorageNamespace: "storage6", CreationDate: now, DefaultBranchID: "main6"}}, | ||
{RepositoryID: "re", Repository: &graveler.Repository{StorageID: "storage", StorageNamespace: "storageNS1", CreationDate: now, DefaultBranchID: "main1"}}, | ||
{RepositoryID: "repo1", Repository: &graveler.Repository{StorageID: "storage1", StorageNamespace: "storageNS2", CreationDate: now, DefaultBranchID: "main2"}}, | ||
{RepositoryID: "repo2", Repository: &graveler.Repository{StorageID: "storage1", StorageNamespace: "storageNS3", CreationDate: now, DefaultBranchID: "main3"}}, | ||
{RepositoryID: "repo22", Repository: &graveler.Repository{StorageID: "storage1", StorageNamespace: "storageNS4", CreationDate: now, DefaultBranchID: "main4"}}, | ||
{RepositoryID: "repo23", Repository: &graveler.Repository{StorageID: "storage1", StorageNamespace: "storageNS5", CreationDate: now, DefaultBranchID: "main5"}}, | ||
{RepositoryID: "repo3", Repository: &graveler.Repository{StorageID: "storage2", StorageNamespace: "storageNS6", CreationDate: now, DefaultBranchID: "main6"}}, | ||
} | ||
type args struct { | ||
limit int | ||
|
@@ -101,12 +101,12 @@ func TestCatalog_ListRepositories(t *testing.T) { | |
searchString: "", | ||
}, | ||
want: []*catalog.Repository{ | ||
{Name: "re", StorageNamespace: "storage1", DefaultBranch: "main1", CreationDate: now}, | ||
{Name: "repo1", StorageNamespace: "storage2", DefaultBranch: "main2", CreationDate: now}, | ||
{Name: "repo2", StorageNamespace: "storage3", DefaultBranch: "main3", CreationDate: now}, | ||
{Name: "repo22", StorageNamespace: "storage4", DefaultBranch: "main4", CreationDate: now}, | ||
{Name: "repo23", StorageNamespace: "storage5", DefaultBranch: "main5", CreationDate: now}, | ||
{Name: "repo3", StorageNamespace: "storage6", DefaultBranch: "main6", CreationDate: now}, | ||
{Name: "re", StorageID: "storage", StorageNamespace: "storageNS1", DefaultBranch: "main1", CreationDate: now}, | ||
{Name: "repo1", StorageID: "storage1", StorageNamespace: "storageNS2", DefaultBranch: "main2", CreationDate: now}, | ||
{Name: "repo2", StorageID: "storage1", StorageNamespace: "storageNS3", DefaultBranch: "main3", CreationDate: now}, | ||
{Name: "repo22", StorageID: "storage1", StorageNamespace: "storageNS4", DefaultBranch: "main4", CreationDate: now}, | ||
{Name: "repo23", StorageID: "storage1", StorageNamespace: "storageNS5", DefaultBranch: "main5", CreationDate: now}, | ||
{Name: "repo3", StorageID: "storage2", StorageNamespace: "storageNS6", DefaultBranch: "main6", CreationDate: now}, | ||
}, | ||
wantHasMore: false, | ||
wantErr: false, | ||
|
@@ -120,7 +120,7 @@ func TestCatalog_ListRepositories(t *testing.T) { | |
searchString: "", | ||
}, | ||
want: []*catalog.Repository{ | ||
{Name: "re", StorageNamespace: "storage1", DefaultBranch: "main1", CreationDate: now}, | ||
{Name: "re", StorageID: "storage", StorageNamespace: "storageNS1", DefaultBranch: "main1", CreationDate: now}, | ||
}, | ||
wantHasMore: true, | ||
wantErr: false, | ||
|
@@ -134,7 +134,7 @@ func TestCatalog_ListRepositories(t *testing.T) { | |
searchString: "", | ||
}, | ||
want: []*catalog.Repository{ | ||
{Name: "repo1", StorageNamespace: "storage2", DefaultBranch: "main2", CreationDate: now}, | ||
{Name: "repo1", StorageID: "storage1", StorageNamespace: "storageNS2", DefaultBranch: "main2", CreationDate: now}, | ||
}, | ||
wantHasMore: true, | ||
wantErr: false, | ||
|
@@ -148,8 +148,8 @@ func TestCatalog_ListRepositories(t *testing.T) { | |
searchString: "", | ||
}, | ||
want: []*catalog.Repository{ | ||
{Name: "repo2", StorageNamespace: "storage3", DefaultBranch: "main3", CreationDate: now}, | ||
{Name: "repo22", StorageNamespace: "storage4", DefaultBranch: "main4", CreationDate: now}, | ||
{Name: "repo2", StorageID: "storage1", StorageNamespace: "storageNS3", DefaultBranch: "main3", CreationDate: now}, | ||
{Name: "repo22", StorageID: "storage1", StorageNamespace: "storageNS4", DefaultBranch: "main4", CreationDate: now}, | ||
}, | ||
wantHasMore: true, | ||
wantErr: false, | ||
|
@@ -163,7 +163,7 @@ func TestCatalog_ListRepositories(t *testing.T) { | |
searchString: "", | ||
}, | ||
want: []*catalog.Repository{ | ||
{Name: "repo1", StorageNamespace: "storage2", DefaultBranch: "main2", CreationDate: now}, | ||
{Name: "repo1", StorageID: "storage1", StorageNamespace: "storageNS2", DefaultBranch: "main2", CreationDate: now}, | ||
}, | ||
wantHasMore: true, | ||
wantErr: false, | ||
|
@@ -177,8 +177,8 @@ func TestCatalog_ListRepositories(t *testing.T) { | |
searchString: "", | ||
}, | ||
want: []*catalog.Repository{ | ||
{Name: "repo23", StorageNamespace: "storage5", DefaultBranch: "main5", CreationDate: now}, | ||
{Name: "repo3", StorageNamespace: "storage6", DefaultBranch: "main6", CreationDate: now}, | ||
{Name: "repo23", StorageID: "storage1", StorageNamespace: "storageNS5", DefaultBranch: "main5", CreationDate: now}, | ||
{Name: "repo3", StorageID: "storage2", StorageNamespace: "storageNS6", DefaultBranch: "main6", CreationDate: now}, | ||
}, | ||
wantHasMore: false, | ||
wantErr: false, | ||
|
@@ -192,9 +192,9 @@ func TestCatalog_ListRepositories(t *testing.T) { | |
searchString: "o2", | ||
}, | ||
want: []*catalog.Repository{ | ||
{Name: "repo2", StorageNamespace: "storage3", DefaultBranch: "main3", CreationDate: now}, | ||
{Name: "repo22", StorageNamespace: "storage4", DefaultBranch: "main4", CreationDate: now}, | ||
{Name: "repo23", StorageNamespace: "storage5", DefaultBranch: "main5", CreationDate: now}, | ||
{Name: "repo2", StorageID: "storage1", StorageNamespace: "storageNS3", DefaultBranch: "main3", CreationDate: now}, | ||
{Name: "repo22", StorageID: "storage1", StorageNamespace: "storageNS4", DefaultBranch: "main4", CreationDate: now}, | ||
{Name: "repo23", StorageID: "storage1", StorageNamespace: "storageNS5", DefaultBranch: "main5", CreationDate: now}, | ||
}, | ||
wantHasMore: false, | ||
wantErr: false, | ||
|
@@ -208,8 +208,8 @@ func TestCatalog_ListRepositories(t *testing.T) { | |
searchString: "o2", | ||
}, | ||
want: []*catalog.Repository{ | ||
{Name: "repo2", StorageNamespace: "storage3", DefaultBranch: "main3", CreationDate: now}, | ||
{Name: "repo22", StorageNamespace: "storage4", DefaultBranch: "main4", CreationDate: now}, | ||
{Name: "repo2", StorageID: "storage1", StorageNamespace: "storageNS3", DefaultBranch: "main3", CreationDate: now}, | ||
{Name: "repo22", StorageID: "storage1", StorageNamespace: "storageNS4", DefaultBranch: "main4", CreationDate: now}, | ||
}, | ||
wantHasMore: true, | ||
wantErr: false, | ||
|
@@ -223,7 +223,7 @@ func TestCatalog_ListRepositories(t *testing.T) { | |
searchString: "o2", | ||
}, | ||
want: []*catalog.Repository{ | ||
{Name: "repo23", StorageNamespace: "storage5", DefaultBranch: "main5", CreationDate: now}, | ||
{Name: "repo23", StorageID: "storage1", StorageNamespace: "storageNS5", DefaultBranch: "main5", CreationDate: now}, | ||
}, | ||
wantHasMore: false, | ||
wantErr: false, | ||
|
@@ -237,8 +237,8 @@ func TestCatalog_ListRepositories(t *testing.T) { | |
searchString: "o2", | ||
}, | ||
want: []*catalog.Repository{ | ||
{Name: "repo22", StorageNamespace: "storage4", DefaultBranch: "main4", CreationDate: now}, | ||
{Name: "repo23", StorageNamespace: "storage5", DefaultBranch: "main5", CreationDate: now}, | ||
{Name: "repo22", StorageID: "storage1", StorageNamespace: "storageNS4", DefaultBranch: "main4", CreationDate: now}, | ||
{Name: "repo23", StorageID: "storage1", StorageNamespace: "storageNS5", DefaultBranch: "main5", CreationDate: now}, | ||
}, | ||
wantHasMore: false, | ||
wantErr: false, | ||
|
@@ -871,6 +871,7 @@ func createPrepareUncommittedTestScenario(t *testing.T, repositoryID string, num | |
repository := &graveler.RepositoryRecord{ | ||
RepositoryID: graveler.RepositoryID(repositoryID), | ||
Repository: &graveler.Repository{ | ||
StorageID: "storage", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is this change relevant at the moment? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We can remove this row and it will still work - There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Lets remove that - and address these tests in the right context (GC). |
||
StorageNamespace: graveler.StorageNamespace("mem://" + repositoryID), | ||
CreationDate: time.Now(), | ||
DefaultBranchID: "main", | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Since only the blockstore is aware of the current configuration (using either
blockstore
orblockstores
), we should have a validation at the adapter level which checks whether this value is "Valid". For OSS the validation should allow empty value (or enforce it - I don't really know if we should allow providing any storage_id in that case)There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Makes sense.
Removed this validation.