From d7ab265b10db278571a64a0ee9fc85873070d68e Mon Sep 17 00:00:00 2001 From: yuzhipeng Date: Tue, 24 Dec 2024 10:40:30 +0800 Subject: [PATCH 01/12] Change commit-msg hook.sh address to right place (#21343) Since hook.sh address has moved from `https://cdn.rawgit.com/tommarshall/git-good-commit/v0.6.1/hook.sh` to `https://cdn.jsdelivr.net/gh/tommarshall/git-good-commit@v0.6.1/hook.sh` fix the address to the moved address. Signed-off-by: yuzhipeng --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fd5dea99345..ffebd1081d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -312,7 +312,7 @@ The commit message should follow the convention on [How to Write a Git Commit Me To help write conformant commit messages, it is recommended to set up the [git-good-commit](https://github.com/tommarshall/git-good-commit) commit hook. Run this command in the Harbor repo's root directory: ```sh -curl https://cdn.rawgit.com/tommarshall/git-good-commit/v0.6.1/hook.sh > .git/hooks/commit-msg && chmod +x .git/hooks/commit-msg +curl https://cdn.jsdelivr.net/gh/tommarshall/git-good-commit@v0.6.1/hook.sh > .git/hooks/commit-msg && chmod +x .git/hooks/commit-msg ``` ### Automated Testing From 462749a6338a3db4ba3a8bdc591fbde41f561c7c Mon Sep 17 00:00:00 2001 From: Slava Lysunkin Date: Thu, 26 Dec 2024 00:08:05 -0600 Subject: [PATCH 02/12] Fixed the type in DTR adapter info (#21357) Signed-off-by: Slava Lysunkin --- src/pkg/reg/adapter/dtr/adapter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/reg/adapter/dtr/adapter.go b/src/pkg/reg/adapter/dtr/adapter.go index 5b6663cb1b2..27d8fbc19d7 100644 --- a/src/pkg/reg/adapter/dtr/adapter.go +++ b/src/pkg/reg/adapter/dtr/adapter.go @@ -74,7 +74,7 @@ func newAdapter(registry *model.Registry) *adapter { // Info returns information of the registry func (a *adapter) Info() (*model.RegistryInfo, error) { return &model.RegistryInfo{ - Type: model.RegistryTypeAzureAcr, + Type: model.RegistryTypeDTR, SupportedResourceTypes: []string{ model.ResourceTypeImage, }, From a14a4d246836d2760d2c9651a9e226270b6265a5 Mon Sep 17 00:00:00 2001 From: Chlins Zhang Date: Fri, 27 Dec 2024 13:52:51 +0800 Subject: [PATCH 03/12] fix: unify the auth data handle to the decode method (#21350) Signed-off-by: chlins --- src/pkg/p2p/preheat/instance/manager.go | 33 ++++- .../p2p/preheat/models/provider/instance.go | 27 ++++ .../preheat/models/provider/instance_test.go | 139 ++++++++++++++++++ 3 files changed, 191 insertions(+), 8 deletions(-) create mode 100644 src/pkg/p2p/preheat/models/provider/instance_test.go diff --git a/src/pkg/p2p/preheat/instance/manager.go b/src/pkg/p2p/preheat/instance/manager.go index 3f4418a1013..6590d2896c3 100644 --- a/src/pkg/p2p/preheat/instance/manager.go +++ b/src/pkg/p2p/preheat/instance/manager.go @@ -16,7 +16,6 @@ package instance import ( "context" - "encoding/json" "github.com/goharbor/harbor/src/lib/q" dao "github.com/goharbor/harbor/src/pkg/p2p/preheat/dao/instance" @@ -119,11 +118,9 @@ func (dm *manager) Get(ctx context.Context, id int64) (*provider.Instance, error if err != nil { return nil, err } - // mapping auth data to auth info. - if len(ins.AuthData) > 0 { - if err := json.Unmarshal([]byte(ins.AuthData), &ins.AuthInfo); err != nil { - return nil, err - } + + if err := ins.Decode(); err != nil { + return nil, err } return ins, nil @@ -131,7 +128,16 @@ func (dm *manager) Get(ctx context.Context, id int64) (*provider.Instance, error // Get implements @Manager.GetByName func (dm *manager) GetByName(ctx context.Context, name string) (*provider.Instance, error) { - return dm.dao.GetByName(ctx, name) + ins, err := dm.dao.GetByName(ctx, name) + if err != nil { + return nil, err + } + + if err := ins.Decode(); err != nil { + return nil, err + } + + return ins, nil } // Count implements @Manager.Count @@ -141,5 +147,16 @@ func (dm *manager) Count(ctx context.Context, query *q.Query) (int64, error) { // List implements @Manager.List func (dm *manager) List(ctx context.Context, query *q.Query) ([]*provider.Instance, error) { - return dm.dao.List(ctx, query) + inss, err := dm.dao.List(ctx, query) + if err != nil { + return nil, err + } + + for i := range inss { + if err := inss[i].Decode(); err != nil { + return nil, err + } + } + + return inss, nil } diff --git a/src/pkg/p2p/preheat/models/provider/instance.go b/src/pkg/p2p/preheat/models/provider/instance.go index 407daf0add6..405d20d0b1c 100644 --- a/src/pkg/p2p/preheat/models/provider/instance.go +++ b/src/pkg/p2p/preheat/models/provider/instance.go @@ -82,6 +82,33 @@ func (ins *Instance) ToJSON() (string, error) { return string(data), nil } +// Decode decodes the instance. +func (ins *Instance) Decode() error { + // decode the auth data. + authInfo, err := decodeAuthData(ins.AuthData) + if err != nil { + return err + } + + if len(authInfo) > 0 { + ins.AuthInfo = authInfo + } + + return nil +} + +// decodeAuthData decodes the auth data. +func decodeAuthData(data string) (map[string]string, error) { + authInfo := make(map[string]string) + if len(data) > 0 { + if err := json.Unmarshal([]byte(data), &authInfo); err != nil { + return nil, errors.Wrap(err, "decode auth data error") + } + } + + return authInfo, nil +} + // TableName ... func (ins *Instance) TableName() string { return "p2p_preheat_instance" diff --git a/src/pkg/p2p/preheat/models/provider/instance_test.go b/src/pkg/p2p/preheat/models/provider/instance_test.go new file mode 100644 index 00000000000..329b5e0b223 --- /dev/null +++ b/src/pkg/p2p/preheat/models/provider/instance_test.go @@ -0,0 +1,139 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInstance_FromJSON(t *testing.T) { + tests := []struct { + name string + json string + wantErr bool + }{ + { + name: "Empty JSON", + json: "", + wantErr: true, + }, + { + name: "Invalid JSON", + json: "{invalid}", + wantErr: true, + }, + { + name: "Valid JSON", + json: `{ + "id": 1, + "name": "test-instance", + "description": "test description", + "vendor": "test-vendor", + "endpoint": "http://test-endpoint", + "auth_mode": "basic", + "auth_info": {"username": "test", "password": "test123"}, + "enabled": true, + "default": false, + "insecure": false, + "setup_timestamp": 1234567890 + }`, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ins := &Instance{} + err := ins.FromJSON(tt.json) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, int64(1), ins.ID) + assert.Equal(t, "test-instance", ins.Name) + } + }) + } +} + +func TestInstance_ToJSON(t *testing.T) { + ins := &Instance{ + ID: 1, + Name: "test-instance", + Description: "test description", + Vendor: "test-vendor", + Endpoint: "http://test-endpoint", + AuthMode: "basic", + AuthInfo: map[string]string{"username": "test", "password": "test123"}, + Enabled: true, + Default: false, + Insecure: false, + } + + jsonStr, err := ins.ToJSON() + assert.NoError(t, err) + + // Verify the JSON can be decoded back + var decoded Instance + err = json.Unmarshal([]byte(jsonStr), &decoded) + assert.NoError(t, err) + assert.Equal(t, ins.ID, decoded.ID) + assert.Equal(t, ins.Name, decoded.Name) + assert.Equal(t, ins.AuthInfo, decoded.AuthInfo) +} + +func TestInstance_Decode(t *testing.T) { + tests := []struct { + name string + authData string + wantErr bool + }{ + { + name: "Empty auth data", + authData: "", + wantErr: false, + }, + { + name: "Invalid auth data", + authData: "{invalid}", + wantErr: true, + }, + { + name: "Valid auth data", + authData: `{"username": "test", "password": "test123"}`, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ins := &Instance{ + AuthData: tt.authData, + } + err := ins.Decode() + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + if tt.authData != "" { + assert.NotEmpty(t, ins.AuthInfo) + } + } + }) + } +} From abaa40ab609c97d9f953e794bd310927151a6a44 Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Fri, 3 Jan 2025 10:58:24 +0800 Subject: [PATCH 04/12] Skip admin and change oidc user not found message more readable (#21061) fixes #21041 Signed-off-by: stonezdj --- src/pkg/oidc/dao/meta.go | 4 ++++ src/server/middleware/security/oidc_cli.go | 12 ++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/pkg/oidc/dao/meta.go b/src/pkg/oidc/dao/meta.go index 37afec31eaa..3ca57f3b748 100644 --- a/src/pkg/oidc/dao/meta.go +++ b/src/pkg/oidc/dao/meta.go @@ -16,6 +16,7 @@ package dao import ( "context" + "fmt" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/lib/errors" @@ -65,6 +66,9 @@ func (md *metaDAO) GetByUsername(ctx context.Context, username string) (*models. } res := &models.OIDCUser{} if err := ormer.Raw(sql, username).QueryRow(res); err != nil { + if errors.Is(err, orm.ErrNoRows) { + return nil, fmt.Errorf("oidc user data with username %s not found", username) + } return nil, err } return res, nil diff --git a/src/server/middleware/security/oidc_cli.go b/src/server/middleware/security/oidc_cli.go index f62ae2049a0..5ceb55c8841 100644 --- a/src/server/middleware/security/oidc_cli.go +++ b/src/server/middleware/security/oidc_cli.go @@ -63,16 +63,20 @@ func (o *oidcCli) Generate(req *http.Request) security.Context { return nil } - info, err := oidc.VerifySecret(ctx, username, secret) + u, err := uctl.GetByName(ctx, username) if err != nil { - logger.Errorf("failed to verify secret, username: %s, error: %v", username, err) + logger.Errorf("failed to get user model, username: %s, error: %v", username, err) return nil } - u, err := uctl.GetByName(ctx, username) + + info, err := oidc.VerifySecret(ctx, username, secret) if err != nil { - logger.Errorf("failed to get user model, username: %s, error: %v", username, err) + if u.UserID != 1 { // skip the admin user + logger.Errorf("failed to verify secret, username: %s, error: %v", username, err) + } return nil } + oidc.InjectGroupsToUser(info, u) logger.Debugf("an OIDC CLI security context generated for request %s %s", req.Method, req.URL.Path) return local.NewSecurityContext(u) From b0c74a0584e8073452e436d67e53b29784016fb0 Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Fri, 3 Jan 2025 14:11:09 +0800 Subject: [PATCH 05/12] Add swagger api and audit_log_ext table model (#21360) add auditlog-ext related api in swagger add audit_log_ext table Signed-off-by: stonezdj --- api/v2.0/swagger.yaml | 131 +++++++++++++++++- .../postgresql/0160_2.13.0_schema.up.sql | 21 +++ src/server/v2.0/handler/auditlog.go | 12 ++ src/server/v2.0/handler/project.go | 8 ++ 4 files changed, 168 insertions(+), 4 deletions(-) diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 23dcec3bb15..f69cd8c340b 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -1723,9 +1723,9 @@ paths: $ref: '#/responses/500' /audit-logs: get: - summary: Get recent logs of the projects which the user is a member of + summary: Get recent logs of projects which the user is a member with project admin role, or return all audit logs for system admin user (deprecated) description: | - This endpoint let user see the recent operation logs of the projects which he is member of + This endpoint let the user see the recent operation logs of projects which the user is a member with project admin role,, or return all audit logs for system admin user, it only query the audit log in previous version. tags: - auditlog operationId: listAuditLogs @@ -1755,10 +1755,63 @@ paths: $ref: '#/responses/401' '500': $ref: '#/responses/500' + /auditlog-exts: + get: + summary: Get recent logs of the projects which the user is a member with project_admin role, or return all audit logs for system admin user + description: | + This endpoint let user see the recent operation logs of the projects which he is member with project_admin role, or return all audit logs for system admin user. + tags: + - auditlog + operationId: listAuditLogExts + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/query' + - $ref: '#/parameters/sort' + - $ref: '#/parameters/page' + - $ref: '#/parameters/pageSize' + responses: + '200': + description: Success + headers: + X-Total-Count: + description: The total count of auditlogs + type: integer + Link: + description: Link refers to the previous page and next page + type: string + schema: + type: array + items: + $ref: '#/definitions/AuditLogExt' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '500': + $ref: '#/responses/500' + /auditlog-exts/events: + get: + summary: Get all event types of audit log + description: | + Get all event types of audit log + tags: + - auditlog + operationId: listAuditLogEventTypes + parameters: + - $ref: '#/parameters/requestId' + responses: + '200': + description: Success + schema: + type: array + items: + $ref: '#/definitions/AuditLogEventType' + '401': + $ref: '#/responses/401' /projects/{project_name}/logs: get: - summary: Get recent logs of the projects - description: Get recent logs of the projects + summary: Get recent logs of the projects (deprecated) + description: Get recent logs of the projects, it only query the previous version's audit log tags: - project operationId: getLogs @@ -1789,6 +1842,40 @@ paths: $ref: '#/responses/401' '500': $ref: '#/responses/500' + /projects/{project_name}/auditlog-exts: + get: + summary: Get recent logs of the projects + description: Get recent logs of the projects + tags: + - project + operationId: getLogExts + parameters: + - $ref: '#/parameters/projectName' + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/query' + - $ref: '#/parameters/sort' + - $ref: '#/parameters/page' + - $ref: '#/parameters/pageSize' + responses: + '200': + description: Success + headers: + X-Total-Count: + description: The total count of auditlogs + type: integer + Link: + description: Link refers to the previous page and next page + type: string + schema: + type: array + items: + $ref: '#/definitions/AuditLogExt' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '500': + $ref: '#/responses/500' /p2p/preheat/providers: get: summary: List P2P providers @@ -6996,6 +7083,42 @@ definitions: format: date-time example: '2006-01-02T15:04:05Z' description: The time when this operation is triggered. + AuditLogExt: + type: object + properties: + id: + type: integer + description: The ID of the audit log entry. + username: + type: string + description: The username of the operator in this log entry. + resource: + type: string + description: Name of the resource in this log entry. + resource_type: + type: string + description: Type of the resource in this log entry. + operation: + type: string + description: The operation against the resource in this log entry. + operation_description: + type: string + description: The operation's detail description + operation_result: + type: boolean + description: the operation's result, true for success, false for fail + op_time: + type: string + format: date-time + example: '2006-01-02T15:04:05Z' + description: The time when this operation is triggered. + AuditLogEventType: + type: object + properties: + event_type: + type: string + description: the event type, such as create_user. + example: create_user Metadata: type: object properties: diff --git a/make/migrations/postgresql/0160_2.13.0_schema.up.sql b/make/migrations/postgresql/0160_2.13.0_schema.up.sql index 49a13a131a8..88efb21b456 100644 --- a/make/migrations/postgresql/0160_2.13.0_schema.up.sql +++ b/make/migrations/postgresql/0160_2.13.0_schema.up.sql @@ -1,2 +1,23 @@ ALTER TABLE p2p_preheat_policy DROP COLUMN IF EXISTS scope; ALTER TABLE p2p_preheat_policy ADD COLUMN IF NOT EXISTS extra_attrs text; + +CREATE TABLE IF NOT EXISTS audit_log_ext +( + id BIGSERIAL PRIMARY KEY NOT NULL, + project_id BIGINT, + operation VARCHAR(50) NULL, + resource_type VARCHAR(50) NULL, + resource VARCHAR(50) NULL, + username VARCHAR(50) NULL, + op_desc VARCHAR(500) NULL, + op_result BOOLEAN DEFAULT true, + payload TEXT NULL, + source_ip VARCHAR(50) NULL, + op_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- add index to the audit_log_ext table +CREATE INDEX IF NOT EXISTS idx_audit_log_ext_op_time ON audit_log_ext (op_time); +CREATE INDEX IF NOT EXISTS idx_audit_log_ext_project_id_optime ON audit_log_ext (project_id, op_time); +CREATE INDEX IF NOT EXISTS idx_audit_log_ext_project_id_resource_type ON audit_log_ext (project_id, resource_type); +CREATE INDEX IF NOT EXISTS idx_audit_log_ext_project_id_operation ON audit_log_ext (project_id, operation); diff --git a/src/server/v2.0/handler/auditlog.go b/src/server/v2.0/handler/auditlog.go index 22540b1aec2..a18105586af 100644 --- a/src/server/v2.0/handler/auditlog.go +++ b/src/server/v2.0/handler/auditlog.go @@ -110,3 +110,15 @@ func (a *auditlogAPI) ListAuditLogs(ctx context.Context, params auditlog.ListAud WithLink(a.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()). WithPayload(auditLogs) } +func (a *auditlogAPI) ListAuditLogExts(ctx context.Context, params auditlog.ListAuditLogExtsParams) middleware.Responder { + // TODO: implement this method + return auditlog.NewListAuditLogExtsOK(). + WithXTotalCount(0). + WithLink(a.Links(ctx, params.HTTPRequest.URL, 0, 0, 0).String()). + WithPayload(nil) +} + +func (a *auditlogAPI) ListAuditLogEventTypes(_ context.Context, _ auditlog.ListAuditLogEventTypesParams) middleware.Responder { + // TODO: implement this method + return auditlog.NewListAuditLogEventTypesOK().WithPayload(nil) +} diff --git a/src/server/v2.0/handler/project.go b/src/server/v2.0/handler/project.go index 57e1dfb7fab..22d432dfefa 100644 --- a/src/server/v2.0/handler/project.go +++ b/src/server/v2.0/handler/project.go @@ -938,3 +938,11 @@ func highestRole(roles []int) int { } return highest } + +func (a *projectAPI) GetLogExts(ctx context.Context, params operation.GetLogExtsParams) middleware.Responder { + // TODO: implement the function + return operation.NewGetLogExtsOK(). + WithXTotalCount(0). + WithLink(a.Links(ctx, params.HTTPRequest.URL, 0, 0, 15).String()). + WithPayload(nil) +} From 60013590380e7b47ccfd06d37f388606d8d1f340 Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Mon, 6 Jan 2025 14:17:42 +0800 Subject: [PATCH 06/12] Update testcase in main branch (#21375) Update robot account e2e testcase for export-cve change update job service schedule testcase switch dockerhub to registry.goharbor.io Signed-off-by: stonezdj --- tests/apitests/python/test_system_permission.py | 8 ++++---- tests/resources/Harbor-Pages/Project_Robot_Account.robot | 4 ++-- tests/resources/TestCaseBody.robot | 4 ++-- tests/robot-cases/Group1-Nightly/Common.robot | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/apitests/python/test_system_permission.py b/tests/apitests/python/test_system_permission.py index 6db69686baa..4e5cb667bf4 100644 --- a/tests/apitests/python/test_system_permission.py +++ b/tests/apitests/python/test_system_permission.py @@ -81,8 +81,8 @@ def call(self): registry_payload = { "insecure": False, "name": "registry-{}".format(random.randint(1000, 9999)), - "type": "docker-hub", - "url": "https://hub.docker.com" + "type": "harbor", + "url": "https://registry.goharbor.io" } create_registry = Permission("{}/registries".format(harbor_base_url), "POST", 201, registry_payload, "id", id_from_header=True) list_registry = Permission("{}/registries".format(harbor_base_url), "GET", 200, registry_payload) @@ -93,8 +93,8 @@ def call(self): registry_ping_payload = { "insecure": False, "name": "registry-{}".format(random.randint(1000, 9999)), - "type": "docker-hub", - "url": "https://hub.docker.com" + "type": "harbor", + "url": "https://registry.goharbor.io" } ping_registry = Permission("{}/registries/ping".format(harbor_base_url), "POST", 200, registry_ping_payload) # registry permissions end diff --git a/tests/resources/Harbor-Pages/Project_Robot_Account.robot b/tests/resources/Harbor-Pages/Project_Robot_Account.robot index fdde3e707ef..2a463322f98 100644 --- a/tests/resources/Harbor-Pages/Project_Robot_Account.robot +++ b/tests/resources/Harbor-Pages/Project_Robot_Account.robot @@ -24,8 +24,8 @@ Create A Project Robot Account ${permission_count}= Create Dictionary ${total}= Set Variable 0 IF '${first_resource}' == 'all' - Set To Dictionary ${permission_count} all= 68 - ${total}= Set Variable 68 + Set To Dictionary ${permission_count} all= 70 + ${total}= Set Variable 70 Retry Element Click //span[text()='Select all'] ELSE FOR ${item} IN @{resources} diff --git a/tests/resources/TestCaseBody.robot b/tests/resources/TestCaseBody.robot index 7bef207c295..5fd078527ea 100644 --- a/tests/resources/TestCaseBody.robot +++ b/tests/resources/TestCaseBody.robot @@ -676,9 +676,9 @@ Create Schedules For Job Service Dashboard Schedules Create An New P2P Preheat Policy ${p2p_policy_name} ${distribution_name} ** ** Scheduled ${schedule_type} ${schedule_cron} # Create a replication policy triggered by schedule Switch to Registries - Create A New Endpoint docker-hub docker-hub${d} ${null} ${null} ${null} Y + Create A New Endpoint harbor goharbor${d} ${null} ${null} ${null} Y Switch To Replication Manage - Create A Rule With Existing Endpoint ${replication_policy_name} pull goharbor/harbor-core image docker-hub${d} ${project_name} filter_tag=dev mode=Scheduled cron=${schedule_cron} + Create A Rule With Existing Endpoint ${replication_policy_name} pull harbor-ci/goharbor/harbor-core image goharbor${d} ${project_name} filter_tag=dev mode=Scheduled cron=${schedule_cron} # Set up a schedule to scan all Switch To Vulnerability Page Set Scan Schedule Custom value=${schedule_cron} diff --git a/tests/robot-cases/Group1-Nightly/Common.robot b/tests/robot-cases/Group1-Nightly/Common.robot index 2509e10c122..c6d82d61fd4 100644 --- a/tests/robot-cases/Group1-Nightly/Common.robot +++ b/tests/robot-cases/Group1-Nightly/Common.robot @@ -723,7 +723,7 @@ Test Case - System Robot Account Cover All Projects Pull image ${ip} '${robot_account_name}' ${token} project${d} hello-world:latest Retry Action Keyword Check System Robot Account API Permission ${robot_account_name} ${token} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} all Retry Action Keyword Check Project Robot Account API Permission ${robot_account_name} ${token} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${project_id} ${project_name} hello-world latest all - Retry Wait Element Visible //clr-dg-row[.//clr-dg-cell[contains(.,'${robot_account_name}')] and .//clr-icon[contains(@class, 'color-green')] and .//button[text()=' 72 PERMISSION(S) '] and .//span[contains(.,'Never Expires')] and .//clr-dg-cell[text()='For testing'] ] + Retry Wait Element Visible //clr-dg-row[.//clr-dg-cell[contains(.,'${robot_account_name}')] and .//clr-icon[contains(@class, 'color-green')] and .//button[text()=' 70 PERMISSION(S) '] and .//span[contains(.,'Never Expires')] and .//clr-dg-cell[text()='For testing'] ] System Robot Account Exist ${robot_account_name} all Close Browser @@ -739,13 +739,13 @@ Test Case - System Robot Account ${project_id}= Set Variable ${words}[-2] Switch To Robot Account ${robot_account_name} ${token}= Create A System Robot Account sys1${d} days days=100 description=For testing cover_all_system_resources=${true} - Retry Wait Element Visible //clr-dg-row[.//clr-dg-cell[contains(.,'${robot_account_name}')] and .//clr-icon[contains(@class, 'color-green')] and .//button[text()=' 72 PERMISSION(S) '] and .//span[contains(.,'99d 23h')] and .//clr-dg-cell[text()='For testing'] and .//clr-dg-cell//span[text()=' None ']] + Retry Wait Element Visible //clr-dg-row[.//clr-dg-cell[contains(.,'${robot_account_name}')] and .//clr-icon[contains(@class, 'color-green')] and .//button[text()=' 70 PERMISSION(S) '] and .//span[contains(.,'99d 23h')] and .//clr-dg-cell[text()='For testing'] and .//clr-dg-cell//span[text()=' None ']] Retry Action Keyword Check System Robot Account API Permission ${robot_account_name} ${token} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} all Retry Action Keyword Check Project Robot Account API Permission ${robot_account_name} ${token} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${project_id} ${project_name} hello-world latest all 1 ${robot_account_name} ${token}= Create A System Robot Account sys2${d} days days=2 description=For testing cover_all_project_resources=${true} Push image ${ip} '${robot_account_name}' ${token} project${d} hello-world:latest - Retry Wait Element Visible //clr-dg-row[.//clr-dg-cell[contains(.,'${robot_account_name}')] and .//clr-icon[contains(@class, 'color-green')] and .//span[text()='All projects with'] and .//button[text()=' 68 PERMISSION(S) '] and .//span[contains(.,'1d 23h')] and .//clr-dg-cell[text()='For testing'] and .//clr-dg-cell//span[text()=' None ']] + Retry Wait Element Visible //clr-dg-row[.//clr-dg-cell[contains(.,'${robot_account_name}')] and .//clr-icon[contains(@class, 'color-green')] and .//span[text()='All projects with'] and .//button[text()=' 70 PERMISSION(S) '] and .//span[contains(.,'1d 23h')] and .//clr-dg-cell[text()='For testing'] and .//clr-dg-cell//span[text()=' None ']] Retry Action Keyword Check System Robot Account API Permission ${robot_account_name} ${token} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} all 1 Retry Action Keyword Check Project Robot Account API Permission ${robot_account_name} ${token} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${project_id} ${project_name} hello-world latest all Close Browser From 875f43b93c684fbc7d548653a9af9dbff34b622e Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Mon, 6 Jan 2025 16:21:52 +0800 Subject: [PATCH 07/12] Add configure item for audit_log_disable (#21368) Add configure item audit_log_disable Signed-off-by: stonezdj --- src/common/const.go | 3 +++ src/lib/config/metadata/metadatalist.go | 1 + src/lib/config/userconfig.go | 13 +++++++++++++ 3 files changed, 17 insertions(+) diff --git a/src/common/const.go b/src/common/const.go index a8166cea3a2..8f3eca1b7c7 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -220,6 +220,9 @@ const ( // ScannerSkipUpdatePullTime ScannerSkipUpdatePullTime = "scanner_skip_update_pulltime" + // AuditLogEventsDisabled + AuditLogEventsDisabled = "audit_log_events_disabled" + // SessionTimeout defines the web session timeout SessionTimeout = "session_timeout" diff --git a/src/lib/config/metadata/metadatalist.go b/src/lib/config/metadata/metadatalist.go index aab4919fd89..d93f71f7707 100644 --- a/src/lib/config/metadata/metadatalist.go +++ b/src/lib/config/metadata/metadatalist.go @@ -191,6 +191,7 @@ var ( {Name: common.AuditLogForwardEndpoint, Scope: UserScope, Group: BasicGroup, EnvKey: "AUDIT_LOG_FORWARD_ENDPOINT", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The endpoint to forward the audit log.`}, {Name: common.SkipAuditLogDatabase, Scope: UserScope, Group: BasicGroup, EnvKey: "SKIP_LOG_AUDIT_DATABASE", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `The option to skip audit log in database`}, {Name: common.ScannerSkipUpdatePullTime, Scope: UserScope, Group: BasicGroup, EnvKey: "SCANNER_SKIP_UPDATE_PULL_TIME", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `The option to skip update pull time for scanner`}, + {Name: common.AuditLogEventsDisabled, Scope: UserScope, Group: BasicGroup, EnvKey: "AUDIT_LOG_EVENTS_DISABLED", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The option to skip audit log for some operations, the key is _ like create_user, delete_user, separated by comma`}, {Name: common.SessionTimeout, Scope: UserScope, Group: BasicGroup, EnvKey: "SESSION_TIMEOUT", DefaultValue: "60", ItemType: &Int64Type{}, Editable: true, Description: `The session timeout in minutes`}, diff --git a/src/lib/config/userconfig.go b/src/lib/config/userconfig.go index 4012097c9e3..1937b3b7e95 100644 --- a/src/lib/config/userconfig.go +++ b/src/lib/config/userconfig.go @@ -261,3 +261,16 @@ func ScannerSkipUpdatePullTime(ctx context.Context) bool { func BannerMessage(ctx context.Context) string { return DefaultMgr().Get(ctx, common.BannerMessage).GetString() } + +// AuditLogEventEnabled returns the audit log enabled setting for a specific event_type, such as delete_user, create_user +func AuditLogEventEnabled(ctx context.Context, eventType string) bool { + disableListStr := DefaultMgr().Get(ctx, common.AuditLogEventsDisabled).GetString() + disableList := strings.Split(disableListStr, ",") + for _, t := range disableList { + tName := strings.TrimSpace(t) + if strings.EqualFold(tName, eventType) { + return false + } + } + return true +} From 8bf710a4059415096a2fe882af1ab019eb0079b6 Mon Sep 17 00:00:00 2001 From: Prasanth Baskar <89722848+bupd@users.noreply.github.com> Date: Mon, 6 Jan 2025 19:37:20 +0530 Subject: [PATCH 08/12] fix: replication rule message in UI (#21299) * updates replication rule confirm message for execution in UI * update en-us-lang and es-es-lang with clear focus on execution * Since different languages have varying interpretations of 'execution' * Its better to update only the English version Signed-off-by: bupd --- src/portal/src/i18n/lang/en-us-lang.json | 4 ++-- src/portal/src/i18n/lang/es-es-lang.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/portal/src/i18n/lang/en-us-lang.json b/src/portal/src/i18n/lang/en-us-lang.json index 2f48978628e..88f756b027a 100644 --- a/src/portal/src/i18n/lang/en-us-lang.json +++ b/src/portal/src/i18n/lang/en-us-lang.json @@ -586,8 +586,8 @@ "FILTER_EXECUTIONS_PLACEHOLDER": "Filter Executions", "DELETION_TITLE": "Confirm Replication Rule Deletion", "DELETION_SUMMARY": "Do you want to delete replication rule {{param}}?", - "REPLICATION_TITLE": "Confirm Rule Replication", - "REPLICATION_SUMMARY": "Do you want to replicate the rule {{param}}?", + "REPLICATION_TITLE": "Confirm Replication Rule Execution", + "REPLICATION_SUMMARY": "Do you want to execute replication rule {{param}}?", "DELETION_TITLE_FAILURE": "failed to delete Rule Deletion", "DELETION_SUMMARY_FAILURE": "have pending/running/retrying status", "REPLICATE_SUMMARY_FAILURE": "have pending/running status", diff --git a/src/portal/src/i18n/lang/es-es-lang.json b/src/portal/src/i18n/lang/es-es-lang.json index 46080e7bf44..b1561fed309 100644 --- a/src/portal/src/i18n/lang/es-es-lang.json +++ b/src/portal/src/i18n/lang/es-es-lang.json @@ -590,8 +590,8 @@ "DELETION_TITLE_FAILURE": "failed to delete Rule Deletion", "DELETION_SUMMARY_FAILURE": "have pending/running/retrying status", "REPLICATE_SUMMARY_FAILURE": "have pending/running status", - "REPLICATION_TITLE": "Confirm Rule replication", - "REPLICATION_SUMMARY": "Do you want to replicate the Rule {{param}}?", + "REPLICATION_TITLE": "Confirm Replication Rule Execution", + "REPLICATION_SUMMARY": "Do you want to execute replication rule {{param}}?", "FILTER_TARGETS_PLACEHOLDER": "Filtrar Endpoints", "DELETION_TITLE_TARGET": "Confirmar Eliminación de Endpoint", "DELETION_SUMMARY_TARGET": "¿Quiere eliminar el endpoint {{param}}?", From 8ca455eb76884e9294a54b935584b3117111569a Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Wed, 8 Jan 2025 17:15:37 +0800 Subject: [PATCH 09/12] Add config max_job_duration_hours for jobservice (#21390) Signed-off-by: stonezdj --- make/harbor.yml.tmpl | 2 ++ make/photon/prepare/templates/jobservice/config.yml.jinja | 2 +- make/photon/prepare/templates/jobservice/env.jinja | 1 + make/photon/prepare/utils/configs.py | 5 +++++ make/photon/prepare/utils/jobservice.py | 2 ++ 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/make/harbor.yml.tmpl b/make/harbor.yml.tmpl index 2125441598f..582b41079b2 100644 --- a/make/harbor.yml.tmpl +++ b/make/harbor.yml.tmpl @@ -135,6 +135,8 @@ trivy: jobservice: # Maximum number of job workers in job service max_job_workers: 10 + # Maximum hours of task duration in job service, default 24 + max_job_duration_hours: 24 # The jobLoggers backend name, only support "STD_OUTPUT", "FILE" and/or "DB" job_loggers: - STD_OUTPUT diff --git a/make/photon/prepare/templates/jobservice/config.yml.jinja b/make/photon/prepare/templates/jobservice/config.yml.jinja index 5698d7517c2..ee36b9885aa 100644 --- a/make/photon/prepare/templates/jobservice/config.yml.jinja +++ b/make/photon/prepare/templates/jobservice/config.yml.jinja @@ -67,7 +67,7 @@ metric: reaper: # the max time to wait for a task to finish, if unfinished after max_update_hours, the task will be mark as error, but the task will continue to run, default value is 24, - max_update_hours: 24 + max_update_hours: {{ max_job_duration_hours }} # the max time for execution in running state without new task created max_dangling_hours: 168 diff --git a/make/photon/prepare/templates/jobservice/env.jinja b/make/photon/prepare/templates/jobservice/env.jinja index 3cfa9e32dda..38e8e1ba6c9 100644 --- a/make/photon/prepare/templates/jobservice/env.jinja +++ b/make/photon/prepare/templates/jobservice/env.jinja @@ -21,6 +21,7 @@ HTTPS_PROXY={{jobservice_https_proxy}} NO_PROXY={{jobservice_no_proxy}} REGISTRY_CREDENTIAL_USERNAME={{registry_username}} REGISTRY_CREDENTIAL_PASSWORD={{registry_password}} +MAX_JOB_DURATION_SECONDS={{max_job_duration_seconds}} {% if metric.enabled %} METRIC_NAMESPACE=harbor diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index aff7867295f..cc66729b332 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -222,6 +222,11 @@ def parse_yaml_config(config_file_path, with_trivy): # jobservice config js_config = configs.get('jobservice') or {} config_dict['max_job_workers'] = js_config["max_job_workers"] + config_dict['max_job_duration_hours'] = js_config["max_job_duration_hours"] or 24 + value = config_dict["max_job_duration_hours"] + if not isinstance(value, int) or value < 24: + config_dict["max_job_duration_hours"] = 24 + config_dict['max_job_duration_seconds'] = config_dict['max_job_duration_hours'] * 3600 config_dict['job_loggers'] = js_config["job_loggers"] config_dict['logger_sweeper_duration'] = js_config["logger_sweeper_duration"] config_dict['jobservice_secret'] = generate_random_string(16) diff --git a/make/photon/prepare/utils/jobservice.py b/make/photon/prepare/utils/jobservice.py index 96bcea51861..f6e1df85a9d 100644 --- a/make/photon/prepare/utils/jobservice.py +++ b/make/photon/prepare/utils/jobservice.py @@ -33,6 +33,8 @@ def prepare_job_service(config_dict): gid=DEFAULT_GID, internal_tls=config_dict['internal_tls'], max_job_workers=config_dict['max_job_workers'], + max_job_duration_hours=config_dict['max_job_duration_hours'], + max_job_duration_seconds=config_dict['max_job_duration_seconds'], job_loggers=config_dict['job_loggers'], logger_sweeper_duration=config_dict['logger_sweeper_duration'], redis_url=config_dict['redis_url_js'], From 12382fa8aea0c0c762fa091c8d8aa287ca1bbc93 Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Fri, 10 Jan 2025 10:56:43 +0800 Subject: [PATCH 10/12] Update prepare to avoid error when max_job_duration_hours not configured (#21395) Signed-off-by: stonezdj --- make/photon/prepare/utils/configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index cc66729b332..cc72bd429ef 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -222,7 +222,7 @@ def parse_yaml_config(config_file_path, with_trivy): # jobservice config js_config = configs.get('jobservice') or {} config_dict['max_job_workers'] = js_config["max_job_workers"] - config_dict['max_job_duration_hours'] = js_config["max_job_duration_hours"] or 24 + config_dict['max_job_duration_hours'] = js_config.get("max_job_duration_hours") or 24 value = config_dict["max_job_duration_hours"] if not isinstance(value, int) or value < 24: config_dict["max_job_duration_hours"] = 24 From 15d17a3338ef9c5daaab09da40f42b758f59f174 Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Fri, 10 Jan 2025 06:36:28 +0100 Subject: [PATCH 11/12] Remove robotV1 from code base (#20958) (#20991) It was deprecated in 2.4.0. Signed-off-by: Samuel Gaist Co-authored-by: Wang Yan --- api/v2.0/swagger.yaml | 154 --------------- src/server/v2.0/handler/handler.go | 1 - src/server/v2.0/handler/robotV1.go | 292 ----------------------------- 3 files changed, 447 deletions(-) delete mode 100644 src/server/v2.0/handler/robotV1.go diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index f69cd8c340b..5f3896b1970 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -2438,160 +2438,6 @@ paths: $ref: '#/responses/404' '500': $ref: '#/responses/500' - /projects/{project_name_or_id}/robots: - get: - summary: Get all robot accounts of specified project - description: Get all robot accounts of specified project - parameters: - - $ref: '#/parameters/requestId' - - $ref: '#/parameters/isResourceName' - - $ref: '#/parameters/projectNameOrId' - - $ref: '#/parameters/page' - - $ref: '#/parameters/pageSize' - - $ref: '#/parameters/query' - - $ref: '#/parameters/sort' - tags: - - robotv1 - operationId: ListRobotV1 - responses: - '200': - description: Success - headers: - X-Total-Count: - description: The total count of robot accounts - type: integer - Link: - description: Link refers to the previous page and next page - type: string - schema: - type: array - items: - $ref: '#/definitions/Robot' - '400': - $ref: '#/responses/400' - '404': - $ref: '#/responses/404' - '500': - $ref: '#/responses/500' - post: - summary: Create a robot account - description: Create a robot account - tags: - - robotv1 - operationId: CreateRobotV1 - parameters: - - $ref: '#/parameters/requestId' - - $ref: '#/parameters/isResourceName' - - $ref: '#/parameters/projectNameOrId' - - name: robot - in: body - description: The JSON object of a robot account. - required: true - schema: - $ref: '#/definitions/RobotCreateV1' - responses: - '201': - description: Created - headers: - X-Request-Id: - description: The ID of the corresponding request for the response - type: string - Location: - description: The location of the resource - type: string - schema: - $ref: '#/definitions/RobotCreated' - '400': - $ref: '#/responses/400' - '401': - $ref: '#/responses/401' - '403': - $ref: '#/responses/403' - '404': - $ref: '#/responses/404' - '500': - $ref: '#/responses/500' - /projects/{project_name_or_id}/robots/{robot_id}: - get: - summary: Get a robot account - description: This endpoint returns specific robot account information by robot ID. - tags: - - robotv1 - operationId: GetRobotByIDV1 - parameters: - - $ref: '#/parameters/requestId' - - $ref: '#/parameters/isResourceName' - - $ref: '#/parameters/projectNameOrId' - - $ref: '#/parameters/robotId' - responses: - '200': - description: Return matched robot information. - schema: - $ref: '#/definitions/Robot' - '401': - $ref: '#/responses/401' - '403': - $ref: '#/responses/403' - '404': - $ref: '#/responses/404' - '500': - $ref: '#/responses/500' - put: - summary: Update status of robot account. - description: Used to disable/enable a specified robot account. - tags: - - robotv1 - operationId: UpdateRobotV1 - parameters: - - $ref: '#/parameters/requestId' - - $ref: '#/parameters/isResourceName' - - $ref: '#/parameters/projectNameOrId' - - $ref: '#/parameters/robotId' - - name: robot - in: body - description: The JSON object of a robot account. - required: true - schema: - $ref: '#/definitions/Robot' - responses: - '200': - $ref: '#/responses/200' - '400': - $ref: '#/responses/400' - '401': - $ref: '#/responses/401' - '403': - $ref: '#/responses/403' - '404': - $ref: '#/responses/404' - '409': - $ref: '#/responses/409' - '500': - $ref: '#/responses/500' - delete: - summary: Delete a robot account - description: This endpoint deletes specific robot account information by robot ID. - tags: - - robotv1 - operationId: DeleteRobotV1 - parameters: - - $ref: '#/parameters/requestId' - - $ref: '#/parameters/isResourceName' - - $ref: '#/parameters/projectNameOrId' - - $ref: '#/parameters/robotId' - responses: - '200': - $ref: '#/responses/200' - '400': - $ref: '#/responses/400' - '401': - $ref: '#/responses/401' - '403': - $ref: '#/responses/403' - '404': - $ref: '#/responses/404' - '500': - $ref: '#/responses/500' '/projects/{project_name_or_id}/immutabletagrules': get: summary: List all immutable tag rules of current project diff --git a/src/server/v2.0/handler/handler.go b/src/server/v2.0/handler/handler.go index 2de7f562f7d..2f770c696bb 100644 --- a/src/server/v2.0/handler/handler.go +++ b/src/server/v2.0/handler/handler.go @@ -44,7 +44,6 @@ func New() http.Handler { PreheatAPI: newPreheatAPI(), IconAPI: newIconAPI(), RobotAPI: newRobotAPI(), - Robotv1API: newRobotV1API(), ReplicationAPI: newReplicationAPI(), RegistryAPI: newRegistryAPI(), SysteminfoAPI: newSystemInfoAPI(), diff --git a/src/server/v2.0/handler/robotV1.go b/src/server/v2.0/handler/robotV1.go deleted file mode 100644 index 885e96e32ce..00000000000 --- a/src/server/v2.0/handler/robotV1.go +++ /dev/null @@ -1,292 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package handler - -import ( - "context" - "fmt" - "regexp" - "strings" - - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - - "github.com/goharbor/harbor/src/common/rbac" - rbac_project "github.com/goharbor/harbor/src/common/rbac/project" - "github.com/goharbor/harbor/src/controller/project" - "github.com/goharbor/harbor/src/controller/robot" - "github.com/goharbor/harbor/src/lib" - "github.com/goharbor/harbor/src/lib/errors" - "github.com/goharbor/harbor/src/lib/log" - "github.com/goharbor/harbor/src/lib/q" - "github.com/goharbor/harbor/src/pkg/permission/types" - pkg_robot "github.com/goharbor/harbor/src/pkg/robot" - pkg "github.com/goharbor/harbor/src/pkg/robot/model" - handler_model "github.com/goharbor/harbor/src/server/v2.0/handler/model" - "github.com/goharbor/harbor/src/server/v2.0/models" - operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/robotv1" -) - -func newRobotV1API() *robotV1API { - return &robotV1API{ - robotCtl: robot.Ctl, - robotMgr: pkg_robot.Mgr, - projectCtr: project.Ctl, - } -} - -type robotV1API struct { - BaseAPI - robotCtl robot.Controller - robotMgr pkg_robot.Manager - projectCtr project.Controller -} - -func (rAPI *robotV1API) CreateRobotV1(ctx context.Context, params operation.CreateRobotV1Params) middleware.Responder { - projectNameOrID := parseProjectNameOrID(params.ProjectNameOrID, params.XIsResourceName) - if err := rAPI.RequireProjectAccess(ctx, projectNameOrID, rbac.ActionCreate, rbac.ResourceRobot); err != nil { - return rAPI.SendError(ctx, err) - } - - if err := rAPI.validate(ctx, params); err != nil { - return rAPI.SendError(ctx, err) - } - - r := &robot.Robot{ - Robot: pkg.Robot{ - Name: params.Robot.Name, - Description: params.Robot.Description, - ExpiresAt: params.Robot.ExpiresAt, - Visible: true, - }, - Level: robot.LEVELPROJECT, - } - - projectName, ok := projectNameOrID.(string) - if !ok { - p, err := rAPI.projectCtr.Get(ctx, projectNameOrID, project.Metadata(false)) - if err != nil { - log.Errorf("failed to get project %s: %v", projectNameOrID, err) - return rAPI.SendError(ctx, err) - } - projectName = p.Name - } - - permission := &robot.Permission{ - Kind: "project", - Namespace: projectName, - } - - var policies []*types.Policy - for _, acc := range params.Robot.Access { - policy := &types.Policy{ - Action: types.Action(acc.Action), - Effect: types.Effect(acc.Effect), - } - res, err := getRawResource(acc.Resource) - if err != nil { - return rAPI.SendError(ctx, err) - } - policy.Resource = types.Resource(res) - policies = append(policies, policy) - } - permission.Access = policies - r.Permissions = append(r.Permissions, permission) - - rid, pwd, err := rAPI.robotCtl.Create(ctx, r) - if err != nil { - return rAPI.SendError(ctx, err) - } - - created, err := rAPI.robotCtl.Get(ctx, rid, nil) - if err != nil { - return rAPI.SendError(ctx, err) - } - - location := fmt.Sprintf("%s/%d", strings.TrimSuffix(params.HTTPRequest.URL.Path, "/"), created.ID) - return operation.NewCreateRobotV1Created().WithLocation(location).WithPayload(&models.RobotCreated{ - ID: created.ID, - Name: created.Name, - Secret: pwd, - CreationTime: strfmt.DateTime(created.CreationTime), - }) -} - -func (rAPI *robotV1API) DeleteRobotV1(ctx context.Context, params operation.DeleteRobotV1Params) middleware.Responder { - projectNameOrID := parseProjectNameOrID(params.ProjectNameOrID, params.XIsResourceName) - if err := rAPI.RequireProjectAccess(ctx, projectNameOrID, rbac.ActionDelete, rbac.ResourceRobot); err != nil { - return rAPI.SendError(ctx, err) - } - - pro, err := rAPI.projectCtr.Get(ctx, projectNameOrID) - if err != nil { - return rAPI.SendError(ctx, err) - } - r, err := rAPI.robotCtl.List(ctx, q.New(q.KeyWords{"ProjectID": pro.ProjectID, "ID": params.RobotID}), &robot.Option{ - WithPermission: true, - }) - if err != nil { - return rAPI.SendError(ctx, err) - } - if len(r) == 0 { - return rAPI.SendError(ctx, errors.NotFoundError(fmt.Errorf("cannot find robot with project id: %d and id: %d", pro.ProjectID, params.RobotID))) - } - - // ignore the not permissions error. - if err := rAPI.robotCtl.Delete(ctx, params.RobotID); err != nil && !errors.IsNotFoundErr(err) { - return rAPI.SendError(ctx, err) - } - return operation.NewDeleteRobotV1OK() -} - -func (rAPI *robotV1API) ListRobotV1(ctx context.Context, params operation.ListRobotV1Params) middleware.Responder { - projectNameOrID := parseProjectNameOrID(params.ProjectNameOrID, params.XIsResourceName) - if err := rAPI.RequireProjectAccess(ctx, projectNameOrID, rbac.ActionList, rbac.ResourceRobot); err != nil { - return rAPI.SendError(ctx, err) - } - - query, err := rAPI.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize) - if err != nil { - return rAPI.SendError(ctx, err) - } - query.Keywords["Visible"] = true - - pro, err := rAPI.projectCtr.Get(ctx, projectNameOrID) - if err != nil { - return rAPI.SendError(ctx, err) - } - - query.Keywords["ProjectID"] = pro.ProjectID - - total, err := rAPI.robotCtl.Count(ctx, query) - if err != nil { - return rAPI.SendError(ctx, err) - } - - robots, err := rAPI.robotCtl.List(ctx, query, &robot.Option{ - WithPermission: true, - }) - if err != nil { - return rAPI.SendError(ctx, err) - } - - var results []*models.Robot - for _, r := range robots { - results = append(results, handler_model.NewRobot(r).ToSwagger()) - } - - return operation.NewListRobotV1OK(). - WithXTotalCount(total). - WithLink(rAPI.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()). - WithPayload(results) -} - -func (rAPI *robotV1API) GetRobotByIDV1(ctx context.Context, params operation.GetRobotByIDV1Params) middleware.Responder { - projectNameOrID := parseProjectNameOrID(params.ProjectNameOrID, params.XIsResourceName) - if err := rAPI.RequireProjectAccess(ctx, projectNameOrID, rbac.ActionRead, rbac.ResourceRobot); err != nil { - return rAPI.SendError(ctx, err) - } - - pro, err := rAPI.projectCtr.Get(ctx, projectNameOrID) - if err != nil { - return rAPI.SendError(ctx, err) - } - - r, err := rAPI.robotCtl.List(ctx, q.New(q.KeyWords{"ProjectID": pro.ProjectID, "ID": params.RobotID}), &robot.Option{ - WithPermission: true, - }) - if err != nil { - return rAPI.SendError(ctx, err) - } - if len(r) == 0 { - return rAPI.SendError(ctx, errors.NotFoundError(fmt.Errorf("cannot find robot with project id: %d and id: %d", pro.ProjectID, params.RobotID))) - } - - return operation.NewGetRobotByIDV1OK().WithPayload(handler_model.NewRobot(r[0]).ToSwagger()) -} - -func (rAPI *robotV1API) UpdateRobotV1(ctx context.Context, params operation.UpdateRobotV1Params) middleware.Responder { - projectNameOrID := parseProjectNameOrID(params.ProjectNameOrID, params.XIsResourceName) - if err := rAPI.RequireProjectAccess(ctx, projectNameOrID, rbac.ActionUpdate, rbac.ResourceRobot); err != nil { - return rAPI.SendError(ctx, err) - } - - pro, err := rAPI.projectCtr.Get(ctx, projectNameOrID) - if err != nil { - return rAPI.SendError(ctx, err) - } - r, err := rAPI.robotCtl.List(ctx, q.New(q.KeyWords{"ProjectID": pro.ProjectID, "ID": params.RobotID}), &robot.Option{ - WithPermission: true, - }) - if err != nil { - return rAPI.SendError(ctx, err) - } - if len(r) == 0 { - return rAPI.SendError(ctx, errors.NotFoundError(fmt.Errorf("cannot find robot with project id: %d and id: %d", pro.ProjectID, params.RobotID))) - } - robot := r[0] - - // for v1 API, only update the disable. - robot.Disabled = params.Robot.Disable - if err := rAPI.robotCtl.Update(ctx, robot, nil); err != nil { - return rAPI.SendError(ctx, err) - } - - return operation.NewUpdateRobotV1OK() -} - -func (rAPI *robotV1API) validate(ctx context.Context, params operation.CreateRobotV1Params) error { - if params.Robot == nil { - return errors.New(nil).WithMessage("bad request no robot").WithCode(errors.BadRequestCode) - } - if len(params.Robot.Access) == 0 { - return errors.New(nil).WithMessage("bad request no access").WithCode(errors.BadRequestCode) - } - - projectNameOrID := parseProjectNameOrID(params.ProjectNameOrID, params.XIsResourceName) - pro, err := rAPI.projectCtr.Get(ctx, projectNameOrID) - if err != nil { - return err - } - - policies := rbac_project.GetPoliciesOfProject(pro.ProjectID) - - mp := map[string]bool{} - for _, policy := range policies { - mp[policy.String()] = true - } - - for _, policy := range params.Robot.Access { - p := &types.Policy{} - if err := lib.JSONCopy(p, policy); err != nil { - log.Warningf("failed to call JSONCopy on robot access policy when validate, error: %v", err) - } - if !mp[p.String()] { - return errors.New(nil).WithMessagef("%s action of %s resource not exist in project %s", policy.Action, policy.Resource, projectNameOrID).WithCode(errors.BadRequestCode) - } - } - - return nil -} - -// /project/1/repository => repository -func getRawResource(resource string) (string, error) { - resourceReg := regexp.MustCompile("^/project/[0-9]+/(?P[a-z-]+)$") - matches := resourceReg.FindStringSubmatch(resource) - if len(matches) <= 1 { - return "", errors.New(nil).WithMessagef("bad resource %s", resource).WithCode(errors.BadRequestCode) - } - return matches[1], nil -} From b0545c05fd0a70c589f21cdce0bf4e0e60842ed8 Mon Sep 17 00:00:00 2001 From: Wang Yan Date: Fri, 10 Jan 2025 17:02:57 +0800 Subject: [PATCH 12/12] bump up swagger (#21396) * bump up swagger Signed-off-by: wang yan * fix gc driver type Signed-off-by: wang yan --------- Signed-off-by: wang yan --- Makefile | 2 +- .../quota/driver/project/project.go | 4 ++-- src/pkg/quota/driver/driver.go | 6 ++--- src/pkg/quota/models/quota.go | 16 ++++++------- src/server/v2.0/handler/model/quota.go | 24 ++++++++++++++++++- src/testing/pkg/quota/driver/driver.go | 10 ++++---- tools/swagger/Dockerfile | 4 ++-- 7 files changed, 44 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 293887145f3..0dd0d18d9f3 100644 --- a/Makefile +++ b/Makefile @@ -291,7 +291,7 @@ lint_apis: $(SPECTRAL) lint ./api/v2.0/swagger.yaml SWAGGER_IMAGENAME=$(IMAGENAMESPACE)/swagger -SWAGGER_VERSION=v0.25.0 +SWAGGER_VERSION=v0.31.0 SWAGGER=$(RUNCONTAINER) ${SWAGGER_IMAGENAME}:${SWAGGER_VERSION} SWAGGER_GENERATE_SERVER=${SWAGGER} generate server --template-dir=$(TOOLSPATH)/swagger/templates --exclude-main --additional-initialism=CVE --additional-initialism=GC --additional-initialism=OIDC SWAGGER_IMAGE_BUILD_CMD=${DOCKERBUILD} -f ${TOOLSPATH}/swagger/Dockerfile --build-arg GOLANG=${GOBUILDIMAGE} --build-arg SWAGGER_VERSION=${SWAGGER_VERSION} -t ${SWAGGER_IMAGENAME}:$(SWAGGER_VERSION) . diff --git a/src/controller/quota/driver/project/project.go b/src/controller/quota/driver/project/project.go index 4f84d0804c7..859deab60b9 100644 --- a/src/controller/quota/driver/project/project.go +++ b/src/controller/quota/driver/project/project.go @@ -62,7 +62,7 @@ func (d *driver) HardLimits(ctx context.Context) types.ResourceList { } } -func (d *driver) Load(ctx context.Context, key string) (dr.RefObject, error) { +func (d *driver) Load(ctx context.Context, key string) (dr.QuotaRefObject, error) { thunk := d.loader.Load(ctx, dataloader.StringKey(key)) result, err := thunk() @@ -75,7 +75,7 @@ func (d *driver) Load(ctx context.Context, key string) (dr.RefObject, error) { return nil, fmt.Errorf("bad result for project: %s", key) } - return dr.RefObject{ + return dr.QuotaRefObject{ "id": project.ProjectID, "name": project.Name, "owner_name": project.OwnerName, diff --git a/src/pkg/quota/driver/driver.go b/src/pkg/quota/driver/driver.go index ea434027d8d..854c60f74ec 100644 --- a/src/pkg/quota/driver/driver.go +++ b/src/pkg/quota/driver/driver.go @@ -26,8 +26,8 @@ var ( drivers = map[string]Driver{} ) -// RefObject type for quota ref object -type RefObject map[string]interface{} +// QuotaRefObject type for quota ref object +type QuotaRefObject map[string]interface{} // Driver the driver for quota type Driver interface { @@ -36,7 +36,7 @@ type Driver interface { // HardLimits returns default resource list HardLimits(ctx context.Context) types.ResourceList // Load returns quota ref object by key - Load(ctx context.Context, key string) (RefObject, error) + Load(ctx context.Context, key string) (QuotaRefObject, error) // Validate validate the hard limits Validate(hardLimits types.ResourceList) error // CalculateUsage calculate quota usage by reference id diff --git a/src/pkg/quota/models/quota.go b/src/pkg/quota/models/quota.go index 9499d19c4bc..619531adb5e 100644 --- a/src/pkg/quota/models/quota.go +++ b/src/pkg/quota/models/quota.go @@ -25,14 +25,14 @@ import ( // Quota quota model for manager type Quota struct { - ID int64 `orm:"pk;auto;column(id)" json:"id"` - Ref driver.RefObject `orm:"-" json:"ref"` - Reference string `orm:"column(reference)" json:"-"` - ReferenceID string `orm:"column(reference_id)" json:"-"` - Hard string `orm:"column(hard);type(jsonb)" json:"-"` - Used string `orm:"column(used);type(jsonb)" json:"-"` - CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` - UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` + ID int64 `orm:"pk;auto;column(id)" json:"id"` + Ref driver.QuotaRefObject `orm:"-" json:"ref"` + Reference string `orm:"column(reference)" json:"-"` + ReferenceID string `orm:"column(reference_id)" json:"-"` + Hard string `orm:"column(hard);type(jsonb)" json:"-"` + Used string `orm:"column(used);type(jsonb)" json:"-"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` HardVersion int64 `orm:"column(hard_version)" json:"-"` UsedVersion int64 `orm:"column(used_version)" json:"-"` diff --git a/src/server/v2.0/handler/model/quota.go b/src/server/v2.0/handler/model/quota.go index 5d025b90425..6f9b6d7de76 100644 --- a/src/server/v2.0/handler/model/quota.go +++ b/src/server/v2.0/handler/model/quota.go @@ -21,6 +21,7 @@ import ( "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/quota" + "github.com/goharbor/harbor/src/pkg/quota/driver" "github.com/goharbor/harbor/src/pkg/quota/types" "github.com/goharbor/harbor/src/server/v2.0/models" ) @@ -54,7 +55,7 @@ func (q *Quota) ToSwagger(ctx context.Context) *models.Quota { return &models.Quota{ ID: q.ID, - Ref: q.Ref, + Ref: NewQuotaRefObject(q.Ref).ToSwagger(), Hard: NewResourceList(hard).ToSwagger(), Used: NewResourceList(used).ToSwagger(), CreationTime: strfmt.DateTime(q.CreationTime), @@ -66,3 +67,24 @@ func (q *Quota) ToSwagger(ctx context.Context) *models.Quota { func NewQuota(quota *quota.Quota) *Quota { return &Quota{Quota: quota} } + +// QuotaRefObject model +type QuotaRefObject struct { + driver.QuotaRefObject +} + +// ToSwagger converts the QuotaRefObject to the swagger model +func (rl *QuotaRefObject) ToSwagger() models.QuotaRefObject { + result := make(map[string]interface{}, len(rl.QuotaRefObject)) + + for name, value := range rl.QuotaRefObject { + result[string(name)] = value + } + + return result +} + +// NewQuotaRefObject new QuotaRefObject instance +func NewQuotaRefObject(qr driver.QuotaRefObject) *QuotaRefObject { + return &QuotaRefObject{QuotaRefObject: qr} +} diff --git a/src/testing/pkg/quota/driver/driver.go b/src/testing/pkg/quota/driver/driver.go index 564ed3a47f8..3db65819923 100644 --- a/src/testing/pkg/quota/driver/driver.go +++ b/src/testing/pkg/quota/driver/driver.go @@ -95,23 +95,23 @@ func (_m *Driver) HardLimits(ctx context.Context) types.ResourceList { } // Load provides a mock function with given fields: ctx, key -func (_m *Driver) Load(ctx context.Context, key string) (driver.RefObject, error) { +func (_m *Driver) Load(ctx context.Context, key string) (driver.QuotaRefObject, error) { ret := _m.Called(ctx, key) if len(ret) == 0 { panic("no return value specified for Load") } - var r0 driver.RefObject + var r0 driver.QuotaRefObject var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string) (driver.RefObject, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, string) (driver.QuotaRefObject, error)); ok { return rf(ctx, key) } - if rf, ok := ret.Get(0).(func(context.Context, string) driver.RefObject); ok { + if rf, ok := ret.Get(0).(func(context.Context, string) driver.QuotaRefObject); ok { r0 = rf(ctx, key) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(driver.RefObject) + r0 = ret.Get(0).(driver.QuotaRefObject) } } diff --git a/tools/swagger/Dockerfile b/tools/swagger/Dockerfile index 1c8f7e8de4e..abbb94f1588 100644 --- a/tools/swagger/Dockerfile +++ b/tools/swagger/Dockerfile @@ -2,7 +2,7 @@ ARG GOLANG FROM ${GOLANG} ARG SWAGGER_VERSION -RUN curl -fsSL -o /usr/bin/swagger https://github.com/go-swagger/go-swagger/releases/download/$SWAGGER_VERSION/swagger_linux_amd64 && chmod +x /usr/bin/swagger +RUN go install github.com/go-swagger/go-swagger/cmd/swagger@$SWAGGER_VERSION -ENTRYPOINT ["/usr/bin/swagger"] +ENTRYPOINT ["swagger"] CMD ["--help"]