Merge branch 'main' into fix-repl-ui

CONTRIBUTING.md

@@ -312,7 +312,7 @@ The commit message should follow the convention on [How to Write a Git Commit Me
 To help write conformant commit messages, it is recommended to set up the [git-good-commit](https://github.com/tommarshall/git-good-commit) commit hook. Run this command in the Harbor repo's root directory:
 
 ```sh
-curl https://cdn.rawgit.com/tommarshall/git-good-commit/v0.6.1/hook.sh > .git/hooks/commit-msg && chmod +x .git/hooks/commit-msg
+curl https://cdn.jsdelivr.net/gh/tommarshall/git-good-commit@v0.6.1/hook.sh > .git/hooks/commit-msg && chmod +x .git/hooks/commit-msg
 ```
 
 ### Automated Testing

Makefile

@@ -291,7 +291,7 @@ lint_apis:
 	$(SPECTRAL) lint ./api/v2.0/swagger.yaml
 
 SWAGGER_IMAGENAME=$(IMAGENAMESPACE)/swagger
-SWAGGER_VERSION=v0.25.0
+SWAGGER_VERSION=v0.31.0
 SWAGGER=$(RUNCONTAINER) ${SWAGGER_IMAGENAME}:${SWAGGER_VERSION}
 SWAGGER_GENERATE_SERVER=${SWAGGER} generate server --template-dir=$(TOOLSPATH)/swagger/templates --exclude-main --additional-initialism=CVE --additional-initialism=GC --additional-initialism=OIDC
 SWAGGER_IMAGE_BUILD_CMD=${DOCKERBUILD} -f ${TOOLSPATH}/swagger/Dockerfile --build-arg GOLANG=${GOBUILDIMAGE} --build-arg SWAGGER_VERSION=${SWAGGER_VERSION} -t ${SWAGGER_IMAGENAME}:$(SWAGGER_VERSION) .

api/v2.0/swagger.yaml

@@ -1723,9 +1723,9 @@ paths:
           $ref: '#/responses/500'
   /audit-logs:
     get:
-      summary: Get recent logs of the projects which the user is a member of
+      summary: Get recent logs of projects which the user is a member with project admin role, or return all audit logs for system admin user (deprecated)
       description: |
-        This endpoint let user see the recent operation logs of the projects which he is member of
+        This endpoint let the user see the recent operation logs of projects which the user is a member with project admin role,, or return all audit logs for system admin user, it only query the audit log in previous version.
       tags:
         - auditlog
       operationId: listAuditLogs
@@ -1755,10 +1755,63 @@ paths:
           $ref: '#/responses/401'
         '500':
           $ref: '#/responses/500'
+  /auditlog-exts:
+    get:
+      summary: Get recent logs of the projects which the user is a member with project_admin role, or return all audit logs for system admin user
+      description: |
+        This endpoint let user see the recent operation logs of the projects which he is member with project_admin role, or return all audit logs for system admin user.
+      tags:
+        - auditlog
+      operationId: listAuditLogExts
+      parameters:
+        - $ref: '#/parameters/requestId'
+        - $ref: '#/parameters/query'
+        - $ref: '#/parameters/sort'
+        - $ref: '#/parameters/page'
+        - $ref: '#/parameters/pageSize'
+      responses:
+        '200':
+          description: Success
+          headers:
+            X-Total-Count:
+              description: The total count of auditlogs
+              type: integer
+            Link:
+              description: Link refers to the previous page and next page
+              type: string
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/AuditLogExt'
+        '400':
+          $ref: '#/responses/400'
+        '401':
+          $ref: '#/responses/401'
+        '500':
+          $ref: '#/responses/500'
+  /auditlog-exts/events:
+    get:
+      summary: Get all event types of audit log
+      description: |
+        Get all event types of audit log
+      tags:
+        - auditlog
+      operationId: listAuditLogEventTypes
+      parameters:
+        - $ref: '#/parameters/requestId'
+      responses:
+        '200':
+          description: Success
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/AuditLogEventType'
+        '401':
+          $ref: '#/responses/401'      
   /projects/{project_name}/logs:
     get:
-      summary: Get recent logs of the projects
-      description: Get recent logs of the projects
+      summary: Get recent logs of the projects (deprecated)
+      description: Get recent logs of the projects, it only query the previous version's audit log
       tags:
         - project
       operationId: getLogs
@@ -1789,6 +1842,40 @@ paths:
           $ref: '#/responses/401'
         '500':
           $ref: '#/responses/500'
+  /projects/{project_name}/auditlog-exts:
+    get:
+      summary: Get recent logs of the projects
+      description: Get recent logs of the projects
+      tags:
+        - project
+      operationId: getLogExts
+      parameters:
+        - $ref: '#/parameters/projectName'
+        - $ref: '#/parameters/requestId'
+        - $ref: '#/parameters/query'
+        - $ref: '#/parameters/sort'
+        - $ref: '#/parameters/page'
+        - $ref: '#/parameters/pageSize'
+      responses:
+        '200':
+          description: Success
+          headers:
+            X-Total-Count:
+              description: The total count of auditlogs
+              type: integer
+            Link:
+              description: Link refers to the previous page and next page
+              type: string
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/AuditLogExt'
+        '400':
+          $ref: '#/responses/400'
+        '401':
+          $ref: '#/responses/401'
+        '500':
+          $ref: '#/responses/500'          
   /p2p/preheat/providers:
     get:
       summary: List P2P providers
@@ -2351,160 +2438,6 @@ paths:
           $ref: '#/responses/404'
         '500':
           $ref: '#/responses/500'
-  /projects/{project_name_or_id}/robots:
-    get:
-      summary: Get all robot accounts of specified project
-      description: Get all robot accounts of specified project
-      parameters:
-        - $ref: '#/parameters/requestId'
-        - $ref: '#/parameters/isResourceName'
-        - $ref: '#/parameters/projectNameOrId'
-        - $ref: '#/parameters/page'
-        - $ref: '#/parameters/pageSize'
-        - $ref: '#/parameters/query'
-        - $ref: '#/parameters/sort'
-      tags:
-        - robotv1
-      operationId: ListRobotV1
-      responses:
-        '200':
-          description: Success
-          headers:
-            X-Total-Count:
-              description: The total count of robot accounts
-              type: integer
-            Link:
-              description: Link refers to the previous page and next page
-              type: string
-          schema:
-            type: array
-            items:
-              $ref: '#/definitions/Robot'
-        '400':
-          $ref: '#/responses/400'
-        '404':
-          $ref: '#/responses/404'
-        '500':
-          $ref: '#/responses/500'
-    post:
-      summary: Create a robot account
-      description: Create a robot account
-      tags:
-        - robotv1
-      operationId: CreateRobotV1
-      parameters:
-        - $ref: '#/parameters/requestId'
-        - $ref: '#/parameters/isResourceName'
-        - $ref: '#/parameters/projectNameOrId'
-        - name: robot
-          in: body
-          description: The JSON object of a robot account.
-          required: true
-          schema:
-            $ref: '#/definitions/RobotCreateV1'
-      responses:
-        '201':
-          description: Created
-          headers:
-            X-Request-Id:
-              description: The ID of the corresponding request for the response
-              type: string
-            Location:
-              description: The location of the resource
-              type: string
-          schema:
-            $ref: '#/definitions/RobotCreated'
-        '400':
-          $ref: '#/responses/400'
-        '401':
-          $ref: '#/responses/401'
-        '403':
-          $ref: '#/responses/403'
-        '404':
-          $ref: '#/responses/404'
-        '500':
-          $ref: '#/responses/500'
-  /projects/{project_name_or_id}/robots/{robot_id}:
-    get:
-      summary: Get a robot account
-      description: This endpoint returns specific robot account information by robot ID.
-      tags:
-        - robotv1
-      operationId: GetRobotByIDV1
-      parameters:
-        - $ref: '#/parameters/requestId'
-        - $ref: '#/parameters/isResourceName'
-        - $ref: '#/parameters/projectNameOrId'
-        - $ref: '#/parameters/robotId'
-      responses:
-        '200':
-          description: Return matched robot information.
-          schema:
-            $ref: '#/definitions/Robot'
-        '401':
-          $ref: '#/responses/401'
-        '403':
-          $ref: '#/responses/403'
-        '404':
-          $ref: '#/responses/404'
-        '500':
-          $ref: '#/responses/500'
-    put:
-      summary: Update status of robot account.
-      description: Used to disable/enable a specified robot account.
-      tags:
-        - robotv1
-      operationId: UpdateRobotV1
-      parameters:
-        - $ref: '#/parameters/requestId'
-        - $ref: '#/parameters/isResourceName'
-        - $ref: '#/parameters/projectNameOrId'
-        - $ref: '#/parameters/robotId'
-        - name: robot
-          in: body
-          description: The JSON object of a robot account.
-          required: true
-          schema:
-            $ref: '#/definitions/Robot'
-      responses:
-        '200':
-          $ref: '#/responses/200'
-        '400':
-          $ref: '#/responses/400'
-        '401':
-          $ref: '#/responses/401'
-        '403':
-          $ref: '#/responses/403'
-        '404':
-          $ref: '#/responses/404'
-        '409':
-          $ref: '#/responses/409'
-        '500':
-          $ref: '#/responses/500'
-    delete:
-      summary: Delete a robot account
-      description: This endpoint deletes specific robot account information by robot ID.
-      tags:
-        - robotv1
-      operationId: DeleteRobotV1
-      parameters:
-        - $ref: '#/parameters/requestId'
-        - $ref: '#/parameters/isResourceName'
-        - $ref: '#/parameters/projectNameOrId'
-        - $ref: '#/parameters/robotId'
-      responses:
-        '200':
-          $ref: '#/responses/200'
-        '400':
-          $ref: '#/responses/400'
-        '401':
-          $ref: '#/responses/401'
-        '403':
-          $ref: '#/responses/403'
-        '404':
-          $ref: '#/responses/404'
-        '500':
-          $ref: '#/responses/500'
   '/projects/{project_name_or_id}/immutabletagrules':
     get:
       summary: List all immutable tag rules of current project
@@ -6996,6 +6929,42 @@ definitions:
         format: date-time
         example: '2006-01-02T15:04:05Z'
         description: The time when this operation is triggered.
+  AuditLogExt:
+    type: object
+    properties:
+      id:
+        type: integer
+        description: The ID of the audit log entry.
+      username:
+        type: string
+        description: The username of the operator in this log entry.
+      resource:
+        type: string
+        description: Name of the resource in this log entry.
+      resource_type:
+        type: string
+        description: Type of the resource in this log entry.
+      operation:
+        type: string
+        description: The operation against the resource in this log entry.
+      operation_description:
+        type: string
+        description: The operation's detail description
+      operation_result:
+        type: boolean
+        description: the operation's result, true for success, false for fail
+      op_time:
+        type: string
+        format: date-time
+        example: '2006-01-02T15:04:05Z'
+        description: The time when this operation is triggered.
+  AuditLogEventType:
+    type: object
+    properties: 
+      event_type: 
+        type: string
+        description: the event type, such as create_user.
+        example: create_user
   Metadata:
     type: object
     properties:

make/harbor.yml.tmpl

@@ -135,6 +135,8 @@ trivy:
 jobservice:
   # Maximum number of job workers in job service
   max_job_workers: 10
+  # Maximum hours of task duration in job service, default 24
+  max_job_duration_hours: 24
   # The jobLoggers backend name, only support "STD_OUTPUT", "FILE" and/or "DB"
   job_loggers:
     - STD_OUTPUT

make/migrations/postgresql/0160_2.13.0_schema.up.sql

@@ -1,2 +1,23 @@
 ALTER TABLE p2p_preheat_policy DROP COLUMN IF EXISTS scope;
 ALTER TABLE p2p_preheat_policy ADD COLUMN IF NOT EXISTS extra_attrs text;
+
+CREATE TABLE IF NOT EXISTS audit_log_ext
+(
+	id BIGSERIAL PRIMARY KEY NOT NULL,
+	project_id BIGINT,
+	operation VARCHAR(50) NULL,
+	resource_type VARCHAR(50) NULL,
+	resource VARCHAR(50) NULL,
+	username VARCHAR(50) NULL,
+	op_desc VARCHAR(500) NULL,
+	op_result BOOLEAN DEFAULT true,
+	payload TEXT NULL,
+	source_ip VARCHAR(50) NULL,
+	op_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+-- add index to the audit_log_ext table
+CREATE INDEX IF NOT EXISTS idx_audit_log_ext_op_time ON audit_log_ext (op_time);
+CREATE INDEX IF NOT EXISTS idx_audit_log_ext_project_id_optime ON audit_log_ext (project_id, op_time);
+CREATE INDEX IF NOT EXISTS idx_audit_log_ext_project_id_resource_type ON audit_log_ext (project_id, resource_type);
+CREATE INDEX IF NOT EXISTS idx_audit_log_ext_project_id_operation ON audit_log_ext (project_id, operation);

make/photon/prepare/templates/jobservice/config.yml.jinja

@@ -67,7 +67,7 @@ metric:
 
 reaper:
   # the max time to wait for a task to finish, if unfinished after max_update_hours, the task will be mark as error, but the task will continue to run, default value is 24,
-  max_update_hours: 24
+  max_update_hours: {{ max_job_duration_hours }}
   # the max time for execution in running state without new task created
   max_dangling_hours: 168
 

make/photon/prepare/templates/jobservice/env.jinja

@@ -21,6 +21,7 @@ HTTPS_PROXY={{jobservice_https_proxy}}
 NO_PROXY={{jobservice_no_proxy}}
 REGISTRY_CREDENTIAL_USERNAME={{registry_username}}
 REGISTRY_CREDENTIAL_PASSWORD={{registry_password}}
+MAX_JOB_DURATION_SECONDS={{max_job_duration_seconds}}
 
 {% if metric.enabled %}
 METRIC_NAMESPACE=harbor