Merge branch 'main' into feat-generic-webhook

Makefile

@@ -17,8 +17,8 @@
 # https://stackoverflow.com/questions/920413/make-error-missing-separator
 # https://tutorialedge.net/golang/makefiles-for-go-developers/
 
-SHA := $(shell if [ -d .git ]; then git show -s --format=%h; else echo "default_SHA"; fi)
-TAG := $(shell if [ -d .git ]; then git tag --points-at HEAD; else echo "default_TAG"; fi)
+SHA ?= $(shell git show -s --format=%h 2>/dev/null || echo "default_SHA")
+TAG ?= $(shell git tag --points-at HEAD 2>/dev/null || echo "default_TAG")
 IMAGE_REPO ?= "apache"
 VERSION = $(TAG)@$(SHA)
 

backend/Makefile

@@ -17,13 +17,12 @@
 # https://stackoverflow.com/questions/920413/make-error-missing-separator
 # https://tutorialedge.net/golang/makefiles-for-go-developers/
 
-SHA := $(shell if [ -d .git ]; then git show -s --format=%h; else echo "default_SHA"; fi)
-TAG := $(shell if [ -d .git ]; then git tag --points-at HEAD; else echo "default_TAG"; fi)
+SHA ?= $(shell git show -s --format=%h 2>/dev/null || echo "default_SHA")
+TAG ?= $(shell git tag --points-at HEAD 2>/dev/null || echo "default_TAG")
 IMAGE_REPO ?= "apache"
 VERSION = $(TAG)@$(SHA)
 PYTHON_DIR ?= "./python"
 
-
 all: build
 
 go-dep:

backend/core/models/blueprint.go

@@ -92,3 +92,12 @@ type SyncPolicy struct {
 	TimeAfter  *time.Time `json:"timeAfter"`
 	TriggerSyncPolicy
 }
+
+type ConnectionTokenCheckResult struct {
+	PluginName   string `json:"pluginName" mapstructure:"pluginName"`
+	ConnectionID uint64 `json:"connectionId" mapstructure:"connectionId"`
+	Success      bool   `json:"success" mapstructure:"success"`
+	Message      string `json:"message" mapstructure:"message"`
+}
+
+type ApiBlueprintConnectionTokenCheck []ConnectionTokenCheckResult

backend/core/models/domainlayer/codequality/cq_issues.go

@@ -26,7 +26,7 @@ type CqIssue struct {
 	domainlayer.DomainEntity
 	Rule                     string `gorm:"type:varchar(255)"`
 	Severity                 string `gorm:"type:varchar(100)"`
-	Component                string `gorm:"type:varchar(255)"`
+	Component                string
 	ProjectKey               string `gorm:"index;type:varchar(100)"` //domain project key
 	Line                     int
 	Status                   string `gorm:"type:varchar(20)"`

backend/core/models/domainlayer/devops/cicd_task.go

@@ -36,6 +36,16 @@ const (
 
 const ENV_NAME_PATTERN = "ENV_NAME_PATTERN"
 
+type TransformDeployment struct {
+	Name string `json:"name"`
+	URL  string `json:"url"`
+}
+
+type TransformDeploymentResponse struct {
+	Total int                   `json:"total"`
+	Data  []TransformDeployment `json:"data"`
+}
+
 type CICDTask struct {
 	domainlayer.DomainEntity
 	Name              string `gorm:"type:varchar(255)"`

backend/core/models/migrationscripts/20240919_increase_cq_issue_component_length.go

@@ -0,0 +1,40 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*increaseCqIssueComponentLength)(nil)
+
+type increaseCqIssueComponentLength struct{}
+
+func (script *increaseCqIssueComponentLength) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().ModifyColumnType("cq_issues", "component", "text")
+}
+
+func (*increaseCqIssueComponentLength) Version() uint64 {
+	return 20240919160242
+}
+
+func (*increaseCqIssueComponentLength) Name() string {
+	return "increase cq_issues.component length to text"
+}

backend/core/models/migrationscripts/register.go

@@ -133,5 +133,6 @@ func All() []plugin.MigrationScript {
 		new(addAssigneeToIncident),
 		new(addIsSubtaskToIssue),
 		new(addIsChildToCicdPipeline),
+		new(increaseCqIssueComponentLength),
 	}
 }

backend/core/models/task.go

@@ -34,7 +34,10 @@ const (
 	TASK_PARTIAL   = "TASK_PARTIAL"
 )
 
-var PendingTaskStatus = []string{TASK_CREATED, TASK_RERUN, TASK_RUNNING}
+var (
+	PendingTaskStatus  = []string{TASK_CREATED, TASK_RERUN, TASK_RUNNING}
+	FinishedTaskStatus = []string{TASK_PARTIAL, TASK_CANCELLED, TASK_FAILED, TASK_COMPLETED}
+)
 
 type TaskProgressDetail struct {
 	TotalSubTasks        int    `json:"totalSubTasks"`

backend/core/plugin/plugin_api.go

@@ -73,6 +73,10 @@ type PluginApi interface {
 	ApiResources() map[string]map[string]ApiResourceHandler
 }
 
+type PluginTestConnectionAPI interface {
+	TestConnection(id uint64) errors.Error
+}
+
 const wrapResponseError = "WRAP_RESPONSE_ERROR"
 
 func WrapTestConnectionErrResp(basicRes context.BasicRes, err errors.Error) errors.Error {

backend/core/plugin/plugin_blueprint.go

@@ -93,6 +93,16 @@ type ProjectMapper interface {
 	MapProject(projectName string, scopes []Scope) (models.PipelinePlan, errors.Error)
 }
 
+type ProjectTokenCheckerConnection struct {
+	PluginName   string
+	ConnectionId uint64
+}
+
+// ProjectTokenChecker is implemented by the plugin org, which generate a task tp check all connection's tokens
+type ProjectTokenChecker interface {
+	MakePipeline(skipCollectors bool, projectName string, scopes []ProjectTokenCheckerConnection) (models.PipelinePlan, errors.Error)
+}
+
 // CompositeDataSourcePluginBlueprintV200 is for unit test
 type CompositeDataSourcePluginBlueprintV200 interface {
 	PluginMeta

backend/core/runner/run_task.go

@@ -370,19 +370,17 @@ func UpdateProgressDetail(basicRes context.BasicRes, taskId uint64, progressDeta
 		progressDetail.TotalRecords = p.Total
 	case plugin.SubTaskIncProgress:
 		progressDetail.FinishedRecords = p.Current
-		// update subtask progress
-		where := dal.Where("task_id = ? and name = ?", taskId, progressDetail.SubTaskName)
-		err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{
-			{ColumnName: "finished_records", Value: progressDetail.FinishedRecords},
-		}, where)
-		if err != nil {
-			basicRes.GetLogger().Error(err, "failed to update _devlake_subtasks progress")
-		}
 	case plugin.SetCurrentSubTask:
 		progressDetail.SubTaskName = p.SubTaskName
 		progressDetail.SubTaskNumber = p.SubTaskNumber
-	default:
-		return
+	}
+	// update subtask progress
+	where := dal.Where("task_id = ? and name = ?", taskId, progressDetail.SubTaskName)
+	err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{
+		{ColumnName: "finished_records", Value: progressDetail.FinishedRecords},
+	}, where)
+	if err != nil {
+		basicRes.GetLogger().Error(err, "failed to update _devlake_subtasks progress")
 	}
 }
 
@@ -418,7 +416,7 @@ func recordSubtask(basicRes context.BasicRes, subtask *models.Subtask) {
 		{ColumnName: "began_at", Value: subtask.BeganAt},
 		{ColumnName: "finished_at", Value: subtask.FinishedAt},
 		{ColumnName: "spent_seconds", Value: subtask.SpentSeconds},
-		{ColumnName: "finished_records", Value: subtask.FinishedRecords},
+		//{ColumnName: "finished_records", Value: subtask.FinishedRecords}, // FinishedRecords is zero always.
 		{ColumnName: "number", Value: subtask.Number},
 	}, where); err != nil {
 		basicRes.GetLogger().Error(err, "error writing subtask %d status to DB: %v", subtask.ID)

backend/helpers/pluginhelper/api/api_extractor_stateful.go

@@ -18,6 +18,7 @@ limitations under the License.
 package api
 
 import (
+	"encoding/json"
 	"reflect"
 
 	"github.com/apache/incubator-devlake/core/dal"
@@ -27,9 +28,10 @@ import (
 )
 
 // StatefulApiExtractorArgs is a struct that contains the arguments for a stateful api extractor
-type StatefulApiExtractorArgs struct {
+type StatefulApiExtractorArgs[InputType any] struct {
 	*SubtaskCommonArgs
-	Extract func(row *RawData) ([]any, errors.Error)
+	BeforeExtract func(issue *InputType, stateManager *SubtaskStateManager) errors.Error
+	Extract       func(body *InputType, row *RawData) ([]any, errors.Error)
 }
 
 // StatefulApiExtractor is a struct that manages the stateful API extraction process.
@@ -41,7 +43,7 @@ type StatefulApiExtractorArgs struct {
 //
 // Example:
 //
-//	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{
+//	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[apiv2models.Issue]{
 //	  SubtaskCommonArgs: &api.SubtaskCommonArgs{
 //	    SubTaskContext: subtaskCtx,
 //	    Table:          RAW_ISSUE_TABLE,
@@ -54,8 +56,17 @@ type StatefulApiExtractorArgs struct {
 //	                            // Ensure that the configuration is serializable and contains only public fields.
 //	                            // It is also recommended that the configuration includes only the necessary fields used by the extractor.
 //	..},
-//	  Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-//	    return extractIssues(data, config, row, userFieldMap)
+//	  BeforeExtract: func(body *IssuesResponse, stateManager *api.SubtaskStateManager) errors.Error {
+//	    if stateManager.IsIncremental() {
+//	      // It is important to delete all existing child-records under DiffSync Mode
+//	      err := db.Delete(
+//	        &models.JiraIssueLabel{},
+//	        dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, body.Id),
+//	      )
+//	    }
+//	    return nil
+//	  },
+//	  Extract: func(apiIssue *apiv2models.Issue, row *api.RawData) ([]interface{}, errors.Error) {
 //	  },
 //	})
 //
@@ -64,25 +75,25 @@ type StatefulApiExtractorArgs struct {
 //	}
 //
 // return extractor.Execute()
-type StatefulApiExtractor struct {
-	*StatefulApiExtractorArgs
+type StatefulApiExtractor[InputType any] struct {
+	*StatefulApiExtractorArgs[InputType]
 	*SubtaskStateManager
 }
 
 // NewStatefulApiExtractor creates a new StatefulApiExtractor
-func NewStatefulApiExtractor(args *StatefulApiExtractorArgs) (*StatefulApiExtractor, errors.Error) {
+func NewStatefulApiExtractor[InputType any](args *StatefulApiExtractorArgs[InputType]) (*StatefulApiExtractor[InputType], errors.Error) {
 	stateManager, err := NewSubtaskStateManager(args.SubtaskCommonArgs)
 	if err != nil {
 		return nil, err
 	}
-	return &StatefulApiExtractor{
+	return &StatefulApiExtractor[InputType]{
 		StatefulApiExtractorArgs: args,
 		SubtaskStateManager:      stateManager,
 	}, nil
 }
 
 // Execute sub-task
-func (extractor *StatefulApiExtractor) Execute() errors.Error {
+func (extractor *StatefulApiExtractor[InputType]) Execute() errors.Error {
 	// load data from database
 	db := extractor.GetDal()
 	logger := extractor.GetLogger()
@@ -135,7 +146,20 @@ func (extractor *StatefulApiExtractor) Execute() errors.Error {
 			return errors.Default.Wrap(err, "error fetching row")
 		}
 
-		results, err := extractor.Extract(row)
+		body := new(InputType)
+		err = errors.Convert(json.Unmarshal(row.Data, body))
+		if err != nil {
+			return err
+		}
+
+		if extractor.BeforeExtract != nil {
+			err = extractor.BeforeExtract(body, extractor.SubtaskStateManager)
+			if err != nil {
+				return err
+			}
+		}
+
+		results, err := extractor.Extract(body, row)
 		if err != nil {
 			return errors.Default.Wrap(err, "error calling plugin Extract implementation")
 		}
@@ -169,4 +193,4 @@ func (extractor *StatefulApiExtractor) Execute() errors.Error {
 	return extractor.SubtaskStateManager.Close()
 }
 
-var _ plugin.SubTask = (*StatefulApiExtractor)(nil)
+var _ plugin.SubTask = (*StatefulApiExtractor[any])(nil)

backend/helpers/pluginhelper/api/api_rawdata.go

@@ -34,7 +34,7 @@ type RawData struct {
 	Data      []byte
 	Url       string
 	Input     json.RawMessage `gorm:"type:json"`
-	CreatedAt time.Time
+	CreatedAt time.Time       `gorm:"index"`
 }
 
 type TaskOptions interface {

backend/helpers/pluginhelper/api/data_convertor_stateful.go

@@ -27,9 +27,10 @@ import (
 
 type StatefulDataConverterArgs[InputType any] struct {
 	*SubtaskCommonArgs
-	Input     func(*SubtaskStateManager) (dal.Rows, errors.Error)
-	Convert   func(row *InputType) ([]any, errors.Error)
-	BatchSize int
+	Input         func(*SubtaskStateManager) (dal.Rows, errors.Error)
+	BeforeConvert func(issue *InputType, stateManager *SubtaskStateManager) errors.Error
+	Convert       func(row *InputType) ([]any, errors.Error)
+	BatchSize     int
 }
 
 // StatefulDataConverter is a struct that manages the stateful data conversion process.
@@ -47,7 +48,7 @@ type StatefulDataConverterArgs[InputType any] struct {
 //
 // Example:
 //
-// converter, err := api.NewStatefulDataConverter[models.JiraIssue](&api.StatefulDataConverterArgs[models.JiraIssue]{
+// converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.JiraIssue]{
 // 	SubtaskCommonArgs: &api.SubtaskCommonArgs{
 // 		SubTaskContext: subtaskCtx,
 // 		Table:          RAW_ISSUE_TABLE,
@@ -78,6 +79,15 @@ type StatefulDataConverterArgs[InputType any] struct {
 // 		}
 // 		return db.Cursor(clauses...)
 // 	},
+//	BeforeConvert: func(jiraIssue *models.GitlabMergeRequest, stateManager *api.SubtaskStateManager) errors.Error {
+//		// It is important to delete all existing child-records under DiffSync Mode
+//		issueId := issueIdGen.Generate(data.Options.ConnectionId, jiraIssue.IssueId)
+//		if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil {
+//			return err
+//		}
+//		...
+//		return nil
+//	},
 // 	Convert: func(jiraIssue *models.JiraIssue) ([]interface{}, errors.Error) {
 // 	},
 // })
@@ -93,7 +103,6 @@ type StatefulDataConverter[InputType any] struct {
 }
 
 func NewStatefulDataConverter[
-	OptType any,
 	InputType any,
 ](
 	args *StatefulDataConverterArgs[InputType],
@@ -146,6 +155,13 @@ func (converter *StatefulDataConverter[InputType]) Execute() errors.Error {
 			return errors.Default.Wrap(err, "error fetching rows")
 		}
 
+		if converter.BeforeConvert != nil {
+			err = converter.BeforeConvert(inputRow, converter.SubtaskStateManager)
+			if err != nil {
+				return err
+			}
+		}
+
 		results, err := converter.Convert(inputRow)
 		if err != nil {
 			return errors.Default.Wrap(err, "error calling Converter plugin implementation")