Skip to content

Commit

Permalink
Continue when failing the concurrency (#1810)
Browse files Browse the repository at this point in the history
  • Loading branch information
chmouel authored Nov 8, 2024
1 parent 7e6c443 commit 2609b4b
Show file tree
Hide file tree
Showing 8 changed files with 401 additions and 72 deletions.
4 changes: 2 additions & 2 deletions pkg/reconciler/finalizer.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func (r *Reconciler) FinalizeKind(ctx context.Context, pr *tektonv1.PipelineRun)
repo.Spec.Merge(r.globalRepo.Spec)
}
logger = logger.With("namespace", repo.Namespace)
next := r.qm.RemoveFromQueue(repo, pr)
next := r.qm.RemoveAndTakeItemFromQueue(repo, pr)
if next != "" {
key := strings.Split(next, "/")
pr, err := r.run.Clients.Tekton.TektonV1().PipelineRuns(key[0]).Get(ctx, key[1], metav1.GetOptions{})
Expand All @@ -54,7 +54,7 @@ func (r *Reconciler) FinalizeKind(ctx context.Context, pr *tektonv1.PipelineRun)
}
if err := r.
updatePipelineRunToInProgress(ctx, logger, repo, pr); err != nil {
logger.Error("failed to update status: ", err)
logger.Errorf("failed to update status: %w", err)
return err
}
return nil
Expand Down
72 changes: 52 additions & 20 deletions pkg/reconciler/queue_pipelineruns.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"strings"

"github.com/openshift-pipelines/pipelines-as-code/pkg/apis/pipelinesascode/keys"
"github.com/openshift-pipelines/pipelines-as-code/pkg/apis/pipelinesascode/v1alpha1"
pacAPIv1alpha1 "github.com/openshift-pipelines/pipelines-as-code/pkg/apis/pipelinesascode/v1alpha1"
"github.com/openshift-pipelines/pipelines-as-code/pkg/kubeinteraction"
"github.com/openshift-pipelines/pipelines-as-code/pkg/sync"
tektonv1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
Expand All @@ -22,52 +22,84 @@ func (r *Reconciler) queuePipelineRun(ctx context.Context, logger *zap.SugaredLo
return nil
}

repoName := pr.GetAnnotations()[keys.Repository]
// check if annotation exist
repoName, exist := pr.GetAnnotations()[keys.Repository]
if !exist {
return fmt.Errorf("no %s annotation found", keys.Repository)
}
if repoName == "" {
return fmt.Errorf("annotation %s is empty", keys.Repository)
}
repo, err := r.repoLister.Repositories(pr.Namespace).Get(repoName)
if err != nil {
// if repository is not found, then skip processing the pipelineRun and return nil
if errors.IsNotFound(err) {
r.qm.RemoveRepository(&v1alpha1.Repository{ObjectMeta: metav1.ObjectMeta{
Name: repoName,
Namespace: pr.Namespace,
}})
r.qm.RemoveRepository(&pacAPIv1alpha1.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: repoName,
Namespace: pr.Namespace,
},
})
return nil
}
return fmt.Errorf("updateError: %w", err)
return fmt.Errorf("error getting PipelineRun: %w", err)
}

// merge local repo with global repo here in order to derive settings from global
// for further concurrency and other operations.
if r.globalRepo, err = r.repoLister.Repositories(r.run.Info.Kube.Namespace).Get(r.run.Info.Controller.GlobalRepository); err == nil && r.globalRepo != nil {
logger.Info("Merging global repository settings with local repository settings")
repo.Spec.Merge(r.globalRepo.Spec)
}

// if concurrency was set and later removed or changed to zero
// then remove pipelineRun from Queue and update pending state to running
if repo.Spec.ConcurrencyLimit != nil && *repo.Spec.ConcurrencyLimit == 0 {
_ = r.qm.RemoveFromQueue(repo, pr)
_ = r.qm.RemoveAndTakeItemFromQueue(repo, pr)
if err := r.updatePipelineRunToInProgress(ctx, logger, repo, pr); err != nil {
return fmt.Errorf("failed to update PipelineRun to in_progress: %w", err)
}
return nil
}

orderedList := sync.FilterPipelineRunByState(ctx, r.run.Clients.Tekton, strings.Split(order, ","), tektonv1.PipelineRunSpecStatusPending, kubeinteraction.StateQueued)
acquired, err := r.qm.AddListToRunningQueue(repo, orderedList)
if err != nil {
return fmt.Errorf("failed to add to queue: %s: %w", pr.GetName(), err)
}
var processed bool
var itered int
maxIterations := 5

for _, prKeys := range acquired {
nsName := strings.Split(prKeys, "/")
pr, err = r.run.Clients.Tekton.TektonV1().PipelineRuns(nsName[0]).Get(ctx, nsName[1], metav1.GetOptions{})
orderedList := sync.FilterPipelineRunByState(ctx, r.run.Clients.Tekton, strings.Split(order, ","), tektonv1.PipelineRunSpecStatusPending, kubeinteraction.StateQueued)
for {
acquired, err := r.qm.AddListToRunningQueue(repo, orderedList)
if err != nil {
logger.Info("failed to get pr with namespace and name: ", nsName[0], nsName[1])
return err
return fmt.Errorf("failed to add to queue: %s: %w", pr.GetName(), err)
}
if err := r.updatePipelineRunToInProgress(ctx, logger, repo, pr); err != nil {
return fmt.Errorf("failed to update pipelineRun to in_progress: %w", err)
if len(acquired) == 0 {
logger.Infof("no new PipelineRun acquired for repo %s", repo.GetName())
break
}

for _, prKeys := range acquired {
nsName := strings.Split(prKeys, "/")
repoKey := sync.RepoKey(repo)
pr, err = r.run.Clients.Tekton.TektonV1().PipelineRuns(nsName[0]).Get(ctx, nsName[1], metav1.GetOptions{})
if err != nil {
logger.Info("failed to get pr with namespace and name: ", nsName[0], nsName[1])
_ = r.qm.RemoveFromQueue(repoKey, prKeys)
} else {
if err := r.updatePipelineRunToInProgress(ctx, logger, repo, pr); err != nil {
logger.Errorf("failed to update pipelineRun to in_progress: %w", err)
_ = r.qm.RemoveFromQueue(repoKey, prKeys)
} else {
processed = true
}
}
}
if processed {
break
}
if itered >= maxIterations {
return fmt.Errorf("max iterations reached of %d times trying to get a pipelinerun started for %s", maxIterations, repo.GetName())
}
itered++
}
return nil
}
210 changes: 210 additions & 0 deletions pkg/reconciler/queue_pipelineruns_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
package reconciler

import (
"fmt"
"testing"

"github.com/openshift-pipelines/pipelines-as-code/pkg/apis/pipelinesascode/keys"
pacv1alpha1 "github.com/openshift-pipelines/pipelines-as-code/pkg/apis/pipelinesascode/v1alpha1"
"github.com/openshift-pipelines/pipelines-as-code/pkg/params"
"github.com/openshift-pipelines/pipelines-as-code/pkg/params/clients"
"github.com/openshift-pipelines/pipelines-as-code/pkg/params/info"
testclient "github.com/openshift-pipelines/pipelines-as-code/pkg/test/clients"
testconcurrency "github.com/openshift-pipelines/pipelines-as-code/pkg/test/concurrency"
tektonv1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
"go.uber.org/zap"
zapobserver "go.uber.org/zap/zaptest/observer"
"gotest.tools/v3/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
rtesting "knative.dev/pkg/reconciler/testing"
)

func TestQueuePipelineRun(t *testing.T) {
tests := []struct {
name string
wantErrString string
wantLog string
pipelineRun *tektonv1.PipelineRun
testRepo *pacv1alpha1.Repository
globalRepo *pacv1alpha1.Repository
runningQueue []string
}{
{
name: "no existing order annotation",
pipelineRun: &tektonv1.PipelineRun{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{},
},
},
},
{
name: "no repo name annotation",
pipelineRun: &tektonv1.PipelineRun{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
keys.ExecutionOrder: "repo/foo1",
},
},
},
wantErrString: fmt.Sprintf("no %s annotation found", keys.Repository),
},
{
name: "empty repo name annotation",
pipelineRun: &tektonv1.PipelineRun{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
keys.ExecutionOrder: "repo/foo1",
keys.Repository: "",
},
},
},
wantErrString: fmt.Sprintf("annotation %s is empty", keys.Repository),
},
{
name: "no repo found",
pipelineRun: &tektonv1.PipelineRun{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
keys.ExecutionOrder: "repo/foo1",
keys.Repository: "foo",
},
},
},
},
{
name: "merging global repository settings",
globalRepo: &pacv1alpha1.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "global",
Namespace: "global",
},
Spec: pacv1alpha1.RepositorySpec{
Settings: &pacv1alpha1.Settings{
PipelineRunProvenance: "somewhere",
},
},
},
runningQueue: []string{},
pipelineRun: &tektonv1.PipelineRun{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
Annotations: map[string]string{
keys.ExecutionOrder: "repo/foo1",
keys.Repository: "test",
},
},
},
testRepo: &pacv1alpha1.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
},
Spec: pacv1alpha1.RepositorySpec{
URL: randomURL,
},
},
wantLog: "Merging global repository settings with local repository settings",
},
{
name: "no new PR acquired",
runningQueue: []string{},
pipelineRun: &tektonv1.PipelineRun{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
Annotations: map[string]string{
keys.ExecutionOrder: "repo/foo1",
keys.Repository: "test",
},
},
},
testRepo: &pacv1alpha1.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
},
Spec: pacv1alpha1.RepositorySpec{
URL: randomURL,
},
},
wantLog: "no new PipelineRun acquired for repo test",
},
{
name: "failed to get PR from the Q after many iterations",
runningQueue: []string{"test/test2"},
pipelineRun: &tektonv1.PipelineRun{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
Annotations: map[string]string{
keys.ExecutionOrder: "repo/foo1",
keys.Repository: "test",
},
},
},
testRepo: &pacv1alpha1.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
},
Spec: pacv1alpha1.RepositorySpec{
URL: randomURL,
},
},
wantLog: "failed to get PR",
wantErrString: "max iterations reached of",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
observer, logcatch := zapobserver.New(zap.InfoLevel)
fakelogger := zap.New(observer).Sugar()
ctx, _ := rtesting.SetupFakeContext(t)
repos := []*pacv1alpha1.Repository{}
if tt.testRepo != nil {
repos = append(repos, tt.testRepo)
}
if tt.globalRepo != nil {
repos = append(repos, tt.globalRepo)
}
testData := testclient.Data{
Repositories: repos,
}
stdata, informers := testclient.SeedTestData(t, ctx, testData)
r := &Reconciler{
qm: testconcurrency.TestQMI{
RunningQueue: tt.runningQueue,
},
repoLister: informers.Repository.Lister(),
run: &params.Run{
Info: info.Info{
Kube: &info.KubeOpts{
Namespace: "global",
},
Controller: &info.ControllerInfo{},
},
Clients: clients.Clients{
PipelineAsCode: stdata.PipelineAsCode,
Tekton: stdata.Pipeline,
Kube: stdata.Kube,
Log: fakelogger,
},
},
}
if tt.globalRepo != nil {
r.run.Info.Controller.GlobalRepository = tt.globalRepo.GetName()
}
err := r.queuePipelineRun(ctx, fakelogger, tt.pipelineRun)
if tt.wantErrString != "" {
assert.ErrorContains(t, err, tt.wantErrString)
return
}
assert.NilError(t, err)

if tt.wantLog != "" {
assert.Assert(t, logcatch.FilterMessage(tt.wantLog).Len() != 0, "We didn't get the expected log message", logcatch.All())
}
})
}
}
17 changes: 12 additions & 5 deletions pkg/reconciler/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ type Reconciler struct {
repoLister pacapi.RepositoryLister
pipelineRunLister tektonv1lister.PipelineRunLister
kinteract kubeinteraction.Interface
qm *sync.QueueManager
qm sync.QueueManagerInterface
metrics *metrics.Recorder
eventEmitter *events.EventEmitter
globalRepo *v1alpha1.Repository
Expand Down Expand Up @@ -198,17 +198,24 @@ func (r *Reconciler) reportFinalStatus(ctx context.Context, logger *zap.SugaredL
}

// remove pipelineRun from Queue and start the next one
next := r.qm.RemoveFromQueue(repo, pr)
if next != "" {
for {
next := r.qm.RemoveAndTakeItemFromQueue(repo, pr)
if next == "" {
break
}
key := strings.Split(next, "/")
pr, err := r.run.Clients.Tekton.TektonV1().PipelineRuns(key[0]).Get(ctx, key[1], metav1.GetOptions{})
if err != nil {
return repo, fmt.Errorf("cannot get pipeline for next in queue: %w", err)
logger.Errorf("cannot get pipeline for next in queue: %w", err)
continue
}

if err := r.updatePipelineRunToInProgress(ctx, logger, repo, pr); err != nil {
return repo, fmt.Errorf("failed to update status: %w", err)
logger.Errorf("failed to update status: %w", err)
_ = r.qm.RemoveFromQueue(sync.RepoKey(repo), sync.PrKey(pr))
continue
}
break
}

if err := r.cleanupPipelineRuns(ctx, logger, pacInfo, repo, pr); err != nil {
Expand Down
Loading

0 comments on commit 2609b4b

Please sign in to comment.