Fix a couple of "Data too long" errors from sonarqube, jira, etc (#7885)

* refactor: remove extraction logic from github graphql collector

* fix: pull requests not being updated

* refactor: update github job extractor for single record structure

* refactor: update github issue extractor to extract single record

* refactor: update github deployment extractor to extract single records

* refactor: update github account/release extractor to adopt single record extraction

* docs: add comments

* fix: github issues not being updated

* fix: gitub deployment unit test

* fix: change jira issue component field to text type

* fix: column type should not be parameterized

* fix: #7852 sonarqube issues component field data too long

* fix: change issue.components to text to avoid data too long error

* docs: fix jira issues component tag

* fix: change bitbucket issue components field to text

* fix: change gitee issue components field to text

* fix: change github issue components field to text

* fix: change gitlab issue components field to text

* fix: #7715 Data too long for column 'project_key'

* fix(framework): update trigger api's request body schema (#7888)

* fix(framework): update trigger api's request body schema

* fix(framework): fix test errors

* fix(framework): fix test errors

* test(jira): add e2e test for custom account field in issues (#7894)

* fix: github graphql collectors are not refetching data in incremental mode (#7878)

* refactor: remove extraction logic from github graphql collector

* fix: pull requests not being updated

* refactor: update github job extractor for single record structure

* refactor: update github issue extractor to extract single record

* refactor: update github deployment extractor to extract single records

* refactor: update github account/release extractor to adopt single record extraction

* docs: add comments

* fix: github issues not being updated

* fix: gitub deployment unit test

* fix: github graphql issue collector should order records by CREATED_AT to avoid data missing

* fix: linting

---------

Co-authored-by: Lynwee <[email protected]>

backend/core/dal/dal.go

@@ -167,6 +167,8 @@ type Dal interface {
 	GetPrimaryKeyFields(t reflect.Type) []reflect.StructField
 	// RenameColumn renames column name for specified table
 	RenameColumn(table, oldColumnName, newColumnName string) errors.Error
+	// ModifyColumnType modifies column type
+	ModifyColumnType(table, columnName, columnType string) errors.Error
 	// DropIndexes drops all specified tables
 	DropIndexes(table string, indexes ...string) errors.Error
 	// Dialect returns the dialect of current database

backend/core/models/domainlayer/codequality/cq_projects.go

@@ -26,7 +26,7 @@ import (
 var _ plugin.Scope = (*CqProject)(nil)
 
 type CqProject struct {
-	domainlayer.DomainEntity
+	domainlayer.DomainEntityExtended
 	Name             string `gorm:"type:varchar(255)"`
 	Qualifier        string `gorm:"type:varchar(255)"`
 	Visibility       string `gorm:"type:varchar(64)"`

backend/core/models/domainlayer/ticket/issue.go

@@ -51,7 +51,7 @@ type Issue struct {
 	Priority                string `gorm:"type:varchar(255)"`
 	Severity                string `gorm:"type:varchar(255)"`
 	Urgency                 string `gorm:"type:varchar(255)"`
-	Component               string `gorm:"type:varchar(255)"`
+	Component               string `gorm:"type:text"`
 	OriginalProject         string `gorm:"type:varchar(255)"`
 	IsSubtask               bool
 }

backend/core/models/migrationscripts/20240813_change_issue_component_type.go

@@ -0,0 +1,40 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*changeIssueComponentType)(nil)
+
+type changeIssueComponentType struct{}
+
+func (script *changeIssueComponentType) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().ModifyColumnType("issues", "components", "text")
+}
+
+func (*changeIssueComponentType) Version() uint64 {
+	return 20240813153901
+}
+
+func (*changeIssueComponentType) Name() string {
+	return "change issues.components type to text"
+}

backend/core/models/migrationscripts/20240813_increase_project_key_length.go

@@ -0,0 +1,40 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*increaseProjectKeyLength)(nil)
+
+type increaseProjectKeyLength struct{}
+
+func (script *increaseProjectKeyLength) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().ModifyColumnType("cq_projects", "project_key", "varchar(500)")
+}
+
+func (*increaseProjectKeyLength) Version() uint64 {
+	return 20240813160242
+}
+
+func (*increaseProjectKeyLength) Name() string {
+	return "increase cq_projects.project_key length to 500"
+}

backend/impls/dalgorm/dalgorm.go

@@ -372,6 +372,24 @@ func (d *Dalgorm) RenameColumn(table, oldColumnName, newColumnName string) error
 	)
 }
 
+// ModifyColumnType modify column type
+func (d *Dalgorm) ModifyColumnType(table, columnName, columnType string) errors.Error {
+	// work around the error `cached plan must not change result type` for postgres
+	// wrap in func(){} to make the linter happy
+	defer func() {
+		_ = d.Exec("SELECT * FROM ? LIMIT 1", clause.Table{Name: table})
+	}()
+	query := "ALTER TABLE ? MODIFY COLUMN ? %s"
+	if d.db.Dialector.Name() == "postgres" {
+		query = "ALTER TABLE ? ALTER COLUMN ? TYPE %s"
+	}
+	return d.Exec(
+		fmt.Sprintf(query, columnType),
+		clause.Table{Name: table},
+		clause.Column{Name: columnName},
+	)
+}
+
 // AllTables returns all tables in the database
 func (d *Dalgorm) AllTables() ([]string, errors.Error) {
 	var tableSql string

backend/plugins/bitbucket/models/issue.go

@@ -45,7 +45,7 @@ type BitbucketIssue struct {
 	BitbucketCreatedAt time.Time
 	BitbucketUpdatedAt time.Time `gorm:"index"`
 	Severity           string    `gorm:"type:varchar(255)"`
-	Component          string    `gorm:"type:varchar(255)"`
+	Component          string    `gorm:"type:text"`
 	common.NoPKModel
 }
 

backend/plugins/bitbucket/models/migrationscripts/20240813_change_issue_component_type.go

@@ -0,0 +1,40 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*changeIssueComponentType)(nil)
+
+type changeIssueComponentType struct{}
+
+func (script *changeIssueComponentType) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().ModifyColumnType("_tool_bitbucket_issues", "components", "text")
+}
+
+func (*changeIssueComponentType) Version() uint64 {
+	return 20240813154323
+}
+
+func (*changeIssueComponentType) Name() string {
+	return "change _tool_bitbucket_issues.components type to text"
+}

backend/plugins/bitbucket/models/migrationscripts/register.go

@@ -41,5 +41,6 @@ func All() []plugin.MigrationScript {
 		new(addBuildNumberToPipelines),
 		new(reCreatBitBucketPipelineSteps),
 		new(addMergedByToPr),
+		new(changeIssueComponentType),
 	}
 }

backend/plugins/gitee/models/issue.go

@@ -44,7 +44,7 @@ type GiteeIssue struct {
 	GiteeCreatedAt  time.Time
 	GiteeUpdatedAt  time.Time `gorm:"index"`
 	Severity        string    `gorm:"type:varchar(255)"`
-	Component       string    `gorm:"type:varchar(255)"`
+	Component       string    `gorm:"type:text"`
 	common.NoPKModel
 }
 

backend/plugins/gitee/models/migrationscripts/20240813_change_issue_component_type.go

@@ -0,0 +1,40 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*changeIssueComponentType)(nil)
+
+type changeIssueComponentType struct{}
+
+func (script *changeIssueComponentType) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().ModifyColumnType("_tool_gitee_issues", "components", "text")
+}
+
+func (*changeIssueComponentType) Version() uint64 {
+	return 20240813154445
+}
+
+func (*changeIssueComponentType) Name() string {
+	return "change _tool_gitee_issues.components type to text"
+}

backend/plugins/gitee/models/migrationscripts/register.go

@@ -27,5 +27,6 @@ func All() []plugin.MigrationScript {
 		new(addInitTables),
 		new(addGiteeCommitAuthorInfo),
 		new(addScopeConfigIdToRepo),
+		new(changeIssueComponentType),
 	}
 }

backend/plugins/github/models/issue.go

@@ -45,7 +45,7 @@ type GithubIssue struct {
 	GithubCreatedAt time.Time
 	GithubUpdatedAt time.Time `gorm:"index"`
 	Severity        string    `gorm:"type:varchar(255)"`
-	Component       string    `gorm:"type:varchar(255)"`
+	Component       string    `gorm:"type:text"`
 	common.NoPKModel
 }
 

backend/plugins/github/models/migrationscripts/20240813_change_issue_component_type.go

@@ -0,0 +1,40 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*changeIssueComponentType)(nil)
+
+type changeIssueComponentType struct{}
+
+func (script *changeIssueComponentType) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().ModifyColumnType("_tool_github_issues", "components", "text")
+}
+
+func (*changeIssueComponentType) Version() uint64 {
+	return 20240813154633
+}
+
+func (*changeIssueComponentType) Name() string {
+	return "change _tool_github_issues.components type to text"
+}

backend/plugins/github/models/migrationscripts/register.go

@@ -53,5 +53,6 @@ func All() []plugin.MigrationScript {
 		new(addMergedByToPr),
 		new(restructReviewer),
 		new(addIsDraftToPr),
+		new(changeIssueComponentType),
 	}
 }

backend/plugins/gitlab/models/issue.go

@@ -45,7 +45,7 @@ type GitlabIssue struct {
 	GitlabCreatedAt time.Time
 	GitlabUpdatedAt time.Time `gorm:"index"`
 	Severity        string    `gorm:"type:varchar(255)"`
-	Component       string    `gorm:"type:varchar(255)"`
+	Component       string    `gorm:"type:text"`
 	TimeEstimate    *int64
 	TotalTimeSpent  *int64
 	common.NoPKModel

backend/plugins/gitlab/models/migrationscripts/20240813_change_issue_component_type.go

@@ -0,0 +1,40 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*changeIssueComponentType)(nil)
+
+type changeIssueComponentType struct{}
+
+func (script *changeIssueComponentType) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().ModifyColumnType("_tool_gitlab_issues", "components", "text")
+}
+
+func (*changeIssueComponentType) Version() uint64 {
+	return 20240813154323
+}
+
+func (*changeIssueComponentType) Name() string {
+	return "change _tool_gitlab_issues.components type to text"
+}

backend/plugins/gitlab/models/migrationscripts/register.go

@@ -50,5 +50,6 @@ func All() []plugin.MigrationScript {
 		new(addWebUrlToGitlabPipelineProject),
 		new(addGitlabAssignee),
 		new(addGitlabAssigneeAndReviewerPrimaryKey),
+		new(changeIssueComponentType),
 	}
 }

backend/plugins/jira/models/issue.go

@@ -62,7 +62,7 @@ type JiraIssue struct {
 	LeadTimeMinutes          *uint
 	StdType                  string `gorm:"type:varchar(255)"`
 	StdStatus                string `gorm:"type:varchar(255)"`
-	Components               string `gorm:"type:varchar(255)"`
+	Components               string `gorm:"type:text"`
 	Subtask                  bool
 	ChangelogTotal           int
 	WorklogTotal             int