From 9f92c243082e8a4f599cc8991dd7772a9ce3ccbb Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 26 Jun 2023 14:17:45 -0600 Subject: [PATCH 1/7] fix(schedules): updated_at automatically updates to any change to row (#894) * fix(schedules): updated_at automatically updates to any change to row * revert docker compose * change config to fields and improve comment * what is your why --- api/schedule/create.go | 2 +- api/schedule/update.go | 2 +- cmd/vela-server/schedule.go | 2 +- database/schedule/interface.go | 2 +- database/schedule/update.go | 19 +++++--- database/schedule/update_test.go | 76 +++++++++++++++++++++++++++++++- 6 files changed, 90 insertions(+), 13 deletions(-) diff --git a/api/schedule/create.go b/api/schedule/create.go index c8eb92741..365df170e 100644 --- a/api/schedule/create.go +++ b/api/schedule/create.go @@ -170,7 +170,7 @@ func CreateSchedule(c *gin.Context) { dbSchedule.SetActive(true) // send API call to update the schedule - err = database.FromContext(c).UpdateSchedule(dbSchedule) + err = database.FromContext(c).UpdateSchedule(dbSchedule, true) if err != nil { retErr := fmt.Errorf("unable to set schedule %s to active: %w", dbSchedule.GetName(), err) diff --git a/api/schedule/update.go b/api/schedule/update.go index 646e55fc2..b0862132a 100644 --- a/api/schedule/update.go +++ b/api/schedule/update.go @@ -123,7 +123,7 @@ func UpdateSchedule(c *gin.Context) { } // update the schedule within the database - err = database.FromContext(c).UpdateSchedule(s) + err = database.FromContext(c).UpdateSchedule(s, true) if err != nil { retErr := fmt.Errorf("unable to update scheduled %s: %w", scheduleName, err) diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index ab0a56b90..a9fd38234 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -350,7 +350,7 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat } // send API call to update schedule for ensuring scheduled_at field is set - err = database.UpdateSchedule(s) + err = database.UpdateSchedule(s, false) if err != nil { return fmt.Errorf("unable to update schedule %s/%s: %w", r.GetFullName(), s.GetName(), err) } diff --git a/database/schedule/interface.go b/database/schedule/interface.go index 8aa14efb5..afa163063 100644 --- a/database/schedule/interface.go +++ b/database/schedule/interface.go @@ -45,5 +45,5 @@ type ScheduleInterface interface { // ListSchedulesForRepo defines a function that gets a list of schedules by repo ID. ListSchedulesForRepo(*library.Repo, int, int) ([]*library.Schedule, int64, error) // UpdateSchedule defines a function that updates an existing schedule. - UpdateSchedule(*library.Schedule) error + UpdateSchedule(*library.Schedule, bool) error } diff --git a/database/schedule/update.go b/database/schedule/update.go index b69f8eff0..dd03f62c5 100644 --- a/database/schedule/update.go +++ b/database/schedule/update.go @@ -2,7 +2,6 @@ // // Use of this source code is governed by the LICENSE file in this repository. -//nolint:dupl // ignore similar code with create.go package schedule import ( @@ -13,7 +12,7 @@ import ( ) // UpdateSchedule updates an existing schedule in the database. -func (e *engine) UpdateSchedule(s *library.Schedule) error { +func (e *engine) UpdateSchedule(s *library.Schedule, fields bool) error { e.logger.WithFields(logrus.Fields{ "schedule": s.GetName(), }).Tracef("updating schedule %s in the database", s.GetName()) @@ -27,9 +26,15 @@ func (e *engine) UpdateSchedule(s *library.Schedule) error { return err } - // send query to the database - return e.client. - Table(constants.TableSchedule). - Save(schedule). - Error + // If "fields" is true, update entire record; otherwise, just update scheduled_at (part of processSchedule) + // + // we do this because Gorm will automatically set `updated_at` with the Save function + // and the `updated_at` field should reflect the last time a user updated the record, rather than the scheduler + if fields { + err = e.client.Table(constants.TableSchedule).Save(schedule).Error + } else { + err = e.client.Table(constants.TableSchedule).Model(schedule).UpdateColumn("scheduled_at", s.GetScheduledAt()).Error + } + + return err } diff --git a/database/schedule/update_test.go b/database/schedule/update_test.go index 05fff4286..2554cefdc 100644 --- a/database/schedule/update_test.go +++ b/database/schedule/update_test.go @@ -11,7 +11,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" ) -func TestSchedule_Engine_UpdateSchedule(t *testing.T) { +func TestSchedule_Engine_UpdateSchedule_Config(t *testing.T) { _repo := testRepo() _repo.SetID(1) _repo.SetOrg("foo") @@ -67,7 +67,79 @@ WHERE "id" = $10`). // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err = test.database.UpdateSchedule(_schedule) + err = test.database.UpdateSchedule(_schedule, true) + + if test.failure { + if err == nil { + t.Errorf("UpdateSchedule for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("UpdateSchedule for %s returned err: %v", test.name, err) + } + }) + } +} + +func TestSchedule_Engine_UpdateSchedule_NotConfig(t *testing.T) { + _repo := testRepo() + _repo.SetID(1) + _repo.SetOrg("foo") + _repo.SetName("bar") + _repo.SetFullName("foo/bar") + + _schedule := testSchedule() + _schedule.SetID(1) + _schedule.SetRepoID(1) + _schedule.SetName("nightly") + _schedule.SetEntry("0 0 * * *") + _schedule.SetCreatedAt(1) + _schedule.SetCreatedBy("user1") + _schedule.SetUpdatedAt(1) + _schedule.SetUpdatedBy("user2") + _schedule.SetScheduledAt(1) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query + _mock.ExpectExec(`UPDATE "schedules" SET "scheduled_at"=$1 WHERE "id" = $2`). + WithArgs(1, 1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateSchedule(_schedule) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err = test.database.UpdateSchedule(_schedule, false) if test.failure { if err == nil { From a6277aba3c8b09f7d8b2539e84594a6eeb491255 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Thu, 29 Jun 2023 09:37:29 -0600 Subject: [PATCH 2/7] fix(compose): use hashicorp/vault docker repo (#897) --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d030743e6..09e729ee8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -143,7 +143,7 @@ services: # # https://www.vaultproject.io/ vault: - image: vault:latest + image: hashicorp/vault:latest container_name: vault command: server -dev networks: From 900608b940587c979297c018d831a7df361818b2 Mon Sep 17 00:00:00 2001 From: David May <49894298+wass3rw3rk@users.noreply.github.com> Date: Thu, 6 Jul 2023 16:31:44 -0500 Subject: [PATCH 3/7] fix(api): support schedule in filter (#900) --- api/build/list_org.go | 29 ++++++++++++++++++++++++++++- api/build/list_repo.go | 9 +++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/api/build/list_org.go b/api/build/list_org.go index 1ccd8b45b..9758bbc6a 100644 --- a/api/build/list_org.go +++ b/api/build/list_org.go @@ -35,6 +35,33 @@ import ( // required: true // type: string // - in: query +// name: event +// description: Filter by build event +// type: string +// enum: +// - comment +// - deployment +// - pull_request +// - push +// - schedule +// - tag +// - in: query +// name: branch +// description: Filter builds by branch +// type: string +// - in: query +// name: status +// description: Filter by build status +// type: string +// enum: +// - canceled +// - error +// - failure +// - killed +// - pending +// - running +// - success +// - in: query // name: page // description: The page of results to retrieve // type: integer @@ -109,7 +136,7 @@ func ListBuildsForOrg(c *gin.Context) { // verify the event provided is a valid event type if event != constants.EventComment && event != constants.EventDeploy && event != constants.EventPush && event != constants.EventPull && - event != constants.EventTag { + event != constants.EventTag && event != constants.EventSchedule { retErr := fmt.Errorf("unable to process event %s: invalid event type provided", event) util.HandleError(c, http.StatusBadRequest, retErr) diff --git a/api/build/list_repo.go b/api/build/list_repo.go index 2dd71fc1a..f11d35b0b 100644 --- a/api/build/list_repo.go +++ b/api/build/list_repo.go @@ -45,11 +45,12 @@ import ( // description: Filter by build event // type: string // enum: -// - push +// - comment +// - deployment // - pull_request +// - push +// - schedule // - tag -// - deployment -// - comment // - in: query // name: commit // description: Filter builds based on the commit hash @@ -159,7 +160,7 @@ func ListBuildsForRepo(c *gin.Context) { // verify the event provided is a valid event type if event != constants.EventComment && event != constants.EventDeploy && event != constants.EventPush && event != constants.EventPull && - event != constants.EventTag { + event != constants.EventTag && event != constants.EventSchedule { retErr := fmt.Errorf("unable to process event %s: invalid event type provided", event) util.HandleError(c, http.StatusBadRequest, retErr) From 71a484dd236d606a9f72643b0a6881137f45d620 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Fri, 7 Jul 2023 14:21:41 -0600 Subject: [PATCH 4/7] fix(api/schedule): make validateEntry more strict and set updated_by using claims (#901) * fix(api/schedule): make validateEntry more strict and set updated_by using claims * rm docker compose update * add test cases to validateEntry --- api/schedule/create.go | 36 +++++++++++++++++++++++------------- api/schedule/create_test.go | 16 ++++++++++++++++ api/schedule/update.go | 5 +++++ 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/api/schedule/create.go b/api/schedule/create.go index 365df170e..f03fa0cea 100644 --- a/api/schedule/create.go +++ b/api/schedule/create.go @@ -209,21 +209,31 @@ func validateEntry(minimum time.Duration, entry string) error { return fmt.Errorf("invalid entry of %s", entry) } - // check the previous occurrence of the entry - prevTime, err := gronx.PrevTick(entry, true) - if err != nil { - return err - } + // iterate 5 times through ticks in an effort to catch scalene entries + tickForward := 5 - // check the next occurrence of the entry - nextTime, err := gronx.NextTick(entry, true) - if err != nil { - return err - } + // start with now + t := time.Now().UTC() + + for i := 0; i < tickForward; i++ { + // check the previous occurrence of the entry + prevTime, err := gronx.PrevTickBefore(entry, t, true) + if err != nil { + return err + } + + // check the next occurrence of the entry + nextTime, err := gronx.NextTickAfter(entry, t, false) + if err != nil { + return err + } + + // ensure the time between previous and next schedule exceeds the minimum duration + if nextTime.Sub(prevTime) < minimum { + return fmt.Errorf("entry needs to occur less frequently than every %s", minimum) + } - // ensure the time between previous and next schedule exceeds the minimum duration - if nextTime.Sub(prevTime) < minimum { - return fmt.Errorf("entry needs to occur less frequently than every %s", minimum) + t = nextTime } return nil diff --git a/api/schedule/create_test.go b/api/schedule/create_test.go index 0ca425e32..a2956f6ca 100644 --- a/api/schedule/create_test.go +++ b/api/schedule/create_test.go @@ -35,6 +35,14 @@ func Test_validateEntry(t *testing.T) { }, wantErr: true, }, + { + name: "exceeds minimum frequency with scalene entry pattern", + args: args{ + minimum: 30 * time.Minute, + entry: "1,2,45 * * * *", + }, + wantErr: true, + }, { name: "meets minimum frequency", args: args{ @@ -51,6 +59,14 @@ func Test_validateEntry(t *testing.T) { }, wantErr: false, }, + { + name: "meets minimum frequency with comma entry pattern", + args: args{ + minimum: 15 * time.Minute, + entry: "0,15,30,45 * * * *", + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/api/schedule/update.go b/api/schedule/update.go index b0862132a..5221d222c 100644 --- a/api/schedule/update.go +++ b/api/schedule/update.go @@ -13,6 +13,7 @@ import ( "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/schedule" + "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" "github.com/go-vela/types/library" "github.com/sirupsen/logrus" @@ -73,6 +74,7 @@ func UpdateSchedule(c *gin.Context) { // capture middleware values r := repo.Retrieve(c) s := schedule.Retrieve(c) + u := user.Retrieve(c) scheduleName := util.PathParameter(c, "schedule") minimumFrequency := c.Value("scheduleminimumfrequency").(time.Duration) @@ -122,6 +124,9 @@ func UpdateSchedule(c *gin.Context) { s.SetEntry(input.GetEntry()) } + // set the updated by field using claims + s.SetUpdatedBy(u.GetName()) + // update the schedule within the database err = database.FromContext(c).UpdateSchedule(s, true) if err != nil { From 563f22636bea9c6e70e8d8980a4de7da1d0d1567 Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Mon, 10 Jul 2023 09:29:41 -0500 Subject: [PATCH 5/7] fix(schedules): criteria for triggering a build (#893) * fix(schedules): ignore trigger for first time schedule * fix(schedules): determine trigger off current UTC time * chore: save work * cleanup: ignore inactive schedules * feat: add interval for schedules * chore: address slack feedback * chore: fix typos * fix: processing timed schedules * fix: processing schedules * fix: typo in comment * chore: address review feedback * temp: add test docker compose * fix: finalize * revert: add test docker compose --------- Co-authored-by: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Co-authored-by: David May <49894298+wass3rw3rk@users.noreply.github.com> --- cmd/vela-server/main.go | 8 ++- cmd/vela-server/schedule.go | 123 +++++++++++++++++++++--------------- cmd/vela-server/server.go | 29 +++++---- 3 files changed, 98 insertions(+), 62 deletions(-) diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 19463fe0b..788f6c26b 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -213,9 +213,15 @@ func main() { &cli.DurationFlag{ EnvVars: []string{"VELA_SCHEDULE_MINIMUM_FREQUENCY", "SCHEDULE_MINIMUM_FREQUENCY"}, Name: "schedule-minimum-frequency", - Usage: "minimum time between each schedule entry", + Usage: "minimum time allowed between each build triggered for a schedule", Value: 1 * time.Hour, }, + &cli.DurationFlag{ + EnvVars: []string{"VELA_SCHEDULE_INTERVAL", "SCHEDULE_INTERVAL"}, + Name: "schedule-interval", + Usage: "interval at which schedules will be processed by the server to trigger builds", + Value: 5 * time.Minute, + }, &cli.StringSliceFlag{ EnvVars: []string{"VELA_SCHEDULE_ALLOWLIST"}, Name: "vela-schedule-allowlist", diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index a9fd38234..08e7867cc 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -24,9 +24,13 @@ import ( "k8s.io/apimachinery/pkg/util/wait" ) -const baseErr = "unable to schedule build" +const ( + scheduleErr = "unable to trigger build for schedule" -func processSchedules(compiler compiler.Engine, database database.Interface, metadata *types.Metadata, queue queue.Service, scm scm.Service) error { + scheduleWait = "waiting to trigger build for schedule" +) + +func processSchedules(start time.Time, compiler compiler.Engine, database database.Interface, metadata *types.Metadata, queue queue.Service, scm scm.Service) error { logrus.Infof("processing active schedules to create builds") // send API call to capture the list of active schedules @@ -37,6 +41,13 @@ func processSchedules(compiler compiler.Engine, database database.Interface, met // iterate through the list of active schedules for _, s := range schedules { + // sleep for 1s - 2s before processing the active schedule + // + // This should prevent multiple servers from processing a schedule at the same time by + // leveraging a base duration along with a standard deviation of randomness a.k.a. + // "jitter". To create the jitter, we use a base duration of 1s with a scale factor of 1.0. + time.Sleep(wait.Jitter(time.Second, 1.0)) + // send API call to capture the schedule // // This is needed to ensure we are not dealing with a stale schedule since we fetch @@ -44,52 +55,79 @@ func processSchedules(compiler compiler.Engine, database database.Interface, met // amount of time to get to the end of the list. schedule, err := database.GetSchedule(s.GetID()) if err != nil { - logrus.WithError(err).Warnf("%s for %s", baseErr, schedule.GetName()) + logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) continue } - // create a variable to track if a build should be triggered based off the schedule - trigger := false + // ignore triggering a build if the schedule is no longer active + if !schedule.GetActive() { + logrus.Tracef("skipping to trigger build for inactive schedule %s", schedule.GetName()) - // check if a build has already been triggered for the schedule - if schedule.GetScheduledAt() == 0 { - // trigger a build for the schedule since one has not already been scheduled - trigger = true - } else { - // parse the previous occurrence of the entry for the schedule - prevTime, err := gronx.PrevTick(schedule.GetEntry(), true) - if err != nil { - logrus.WithError(err).Warnf("%s for %s", baseErr, schedule.GetName()) + continue + } - continue - } + // capture the last time a build was triggered for the schedule in UTC + scheduled := time.Unix(schedule.GetScheduledAt(), 0).UTC() - // parse the next occurrence of the entry for the schedule - nextTime, err := gronx.NextTick(schedule.GetEntry(), true) - if err != nil { - logrus.WithError(err).Warnf("%s for %s", baseErr, schedule.GetName()) + // capture the previous occurrence of the entry rounded to the nearest whole interval + // + // i.e. if it's 4:02 on five minute intervals, this will be 4:00 + prevTime, err := gronx.PrevTick(schedule.GetEntry(), true) + if err != nil { + logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) - continue - } + continue + } - // parse the UNIX timestamp from when the last build was triggered for the schedule - t := time.Unix(schedule.GetScheduledAt(), 0).UTC() + // capture the next occurrence of the entry after the last schedule rounded to the nearest whole interval + // + // i.e. if it's 4:02 on five minute intervals, this will be 4:05 + nextTime, err := gronx.NextTickAfter(schedule.GetEntry(), scheduled, true) + if err != nil { + logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) - // check if the time since the last triggered build is greater than the entry duration for the schedule - if time.Since(t) > nextTime.Sub(prevTime) { - // trigger a build for the schedule since it has not previously ran - trigger = true - } + continue } - if trigger && schedule.GetActive() { - err = processSchedule(schedule, compiler, database, metadata, queue, scm) - if err != nil { - logrus.WithError(err).Warnf("%s for %s", baseErr, schedule.GetName()) + // check if we should wait to trigger a build for the schedule + // + // The current time must be after the next occurrence of the schedule. + if !time.Now().After(nextTime) { + logrus.Tracef("%s %s: current time not past next occurrence", scheduleWait, schedule.GetName()) - continue - } + continue + } + + // check if we should wait to trigger a build for the schedule + // + // The previous occurrence of the schedule must be after the starting time of processing schedules. + if !prevTime.After(start) { + logrus.Tracef("%s %s: previous occurence not after starting point", scheduleWait, schedule.GetName()) + + continue + } + + // update the scheduled_at field with the current timestamp + // + // This should help prevent multiple servers from processing a schedule at the same time + // by updating the schedule with a new timestamp to reflect the current state. + schedule.SetScheduledAt(time.Now().UTC().Unix()) + + // send API call to update schedule for ensuring scheduled_at field is set + err = database.UpdateSchedule(schedule, false) + if err != nil { + logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) + + continue + } + + // process the schedule and trigger a new build + err = processSchedule(schedule, compiler, database, metadata, queue, scm) + if err != nil { + logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) + + continue } } @@ -98,13 +136,6 @@ func processSchedules(compiler compiler.Engine, database database.Interface, met //nolint:funlen // ignore function length and number of statements func processSchedule(s *library.Schedule, compiler compiler.Engine, database database.Interface, metadata *types.Metadata, queue queue.Service, scm scm.Service) error { - // sleep for 1s - 3s before processing the schedule - // - // This should prevent multiple servers from processing a schedule at the same time by - // leveraging a base duration along with a standard deviation of randomness a.k.a. - // "jitter". To create the jitter, we use a base duration of 1s with a scale factor of 3.0. - time.Sleep(wait.Jitter(time.Second, 3.0)) - // send API call to capture the repo for the schedule r, err := database.GetRepo(s.GetRepoID()) if err != nil { @@ -337,8 +368,6 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat return err } - s.SetScheduledAt(time.Now().UTC().Unix()) - // break the loop because everything was successful break } // end of retry loop @@ -349,12 +378,6 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat return fmt.Errorf("unable to update repo %s: %w", r.GetFullName(), err) } - // send API call to update schedule for ensuring scheduled_at field is set - err = database.UpdateSchedule(s, false) - if err != nil { - return fmt.Errorf("unable to update schedule %s/%s: %w", r.GetFullName(), s.GetName(), err) - } - // send API call to capture the triggered build b, err = database.GetBuildForRepo(r, b.GetNumber()) if err != nil { diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 0d95e88e5..689ba03c2 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -176,23 +176,30 @@ func server(c *cli.Context) error { g.Go(func() error { logrus.Info("starting scheduler") for { - // cut the configured minimum frequency duration for schedules in half + // track the starting time for when the server begins processing schedules // - // We need to sleep for some amount of time before we attempt to process schedules - // setup in the database. Since the minimum frequency is configurable, we cut it in - // half and use that as the base duration to determine how long to sleep for. - base := c.Duration("schedule-minimum-frequency") / 2 - logrus.Infof("sleeping for %v before scheduling builds", base) + // This will be used to control which schedules will have a build triggered based + // off the configured entry and last time a build was triggered for the schedule. + start := time.Now().UTC() - // sleep for a duration of time before processing schedules + // capture the interval of time to wait before processing schedules // + // We need to sleep for some amount of time before we attempt to process schedules + // setup in the database. Since the schedule interval is configurable, we use that + // as the base duration to determine how long to sleep for. + interval := c.Duration("schedule-interval") + // This should prevent multiple servers from processing schedules at the same time by // leveraging a base duration along with a standard deviation of randomness a.k.a. - // "jitter". To create the jitter, we use the configured minimum frequency duration - // along with a scale factor of 0.1. - time.Sleep(wait.Jitter(base, 0.1)) + // "jitter". To create the jitter, we use the configured schedule interval duration + // along with a scale factor of 0.5. + jitter := wait.Jitter(interval, 0.5) + + logrus.Infof("sleeping for %v before scheduling builds", jitter) + // sleep for a duration of time before processing schedules + time.Sleep(jitter) - err = processSchedules(compiler, database, metadata, queue, scm) + err = processSchedules(start, compiler, database, metadata, queue, scm) if err != nil { logrus.WithError(err).Warn("unable to process schedules") } else { From 3f0c184e0ce446e84c9b55ca5518659f76032b42 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 10 Jul 2023 10:27:35 -0600 Subject: [PATCH 6/7] fix(db tests): use lenient timestamp check for time.Now query matching (#902) Co-authored-by: dave vader <48764154+plyr4@users.noreply.github.com> --- database/build/build_test.go | 15 +++++++++++++++ database/build/clean_test.go | 3 +-- database/schedule/schedule_test.go | 20 ++++++++++++++++++++ database/schedule/update_test.go | 3 +-- database/secret/secret_test.go | 15 +++++++++++++++ database/secret/update_test.go | 7 +++---- database/service/clean_test.go | 3 +-- database/service/service_test.go | 22 ++++++++++++++++++++++ database/step/clean_test.go | 3 +-- database/step/step_test.go | 22 ++++++++++++++++++++++ 10 files changed, 101 insertions(+), 12 deletions(-) diff --git a/database/build/build_test.go b/database/build/build_test.go index 6304cd167..070744260 100644 --- a/database/build/build_test.go +++ b/database/build/build_test.go @@ -8,6 +8,7 @@ import ( "database/sql/driver" "reflect" "testing" + "time" "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/types/library" @@ -266,3 +267,17 @@ type AnyArgument struct{} func (a AnyArgument) Match(v driver.Value) bool { return true } + +// NowTimestamp is used to test whether timestamps get updated correctly to the current time with lenience. +type NowTimestamp struct{} + +// Match satisfies sqlmock.Argument interface. +func (t NowTimestamp) Match(v driver.Value) bool { + ts, ok := v.(int64) + if !ok { + return false + } + now := time.Now().Unix() + + return now-ts < 10 +} diff --git a/database/build/clean_test.go b/database/build/clean_test.go index 05b606b3f..ff6e36556 100644 --- a/database/build/clean_test.go +++ b/database/build/clean_test.go @@ -7,7 +7,6 @@ package build import ( "reflect" "testing" - "time" "github.com/DATA-DOG/go-sqlmock" ) @@ -48,7 +47,7 @@ func TestBuild_Engine_CleanBuilds(t *testing.T) { // ensure the mock expects the name query _mock.ExpectExec(`UPDATE "builds" SET "status"=$1,"error"=$2,"finished"=$3,"deploy_payload"=$4 WHERE created < $5 AND (status = 'running' OR status = 'pending')`). - WithArgs("error", "msg", time.Now().UTC().Unix(), AnyArgument{}, 3). + WithArgs("error", "msg", NowTimestamp{}, AnyArgument{}, 3). WillReturnResult(sqlmock.NewResult(1, 2)) _sqlite := testSqlite(t) diff --git a/database/schedule/schedule_test.go b/database/schedule/schedule_test.go index c86f1f353..fc4b4c6e1 100644 --- a/database/schedule/schedule_test.go +++ b/database/schedule/schedule_test.go @@ -5,8 +5,10 @@ package schedule import ( + "database/sql/driver" "reflect" "testing" + "time" "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/types/library" @@ -210,3 +212,21 @@ func testRepo() *library.Repo { AllowComment: new(bool), } } + +// This will be used with the github.com/DATA-DOG/go-sqlmock library to compare values +// that are otherwise not easily compared. These typically would be values generated +// before adding or updating them in the database. +// +// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime +type NowTimestamp struct{} + +// Match satisfies sqlmock.Argument interface. +func (t NowTimestamp) Match(v driver.Value) bool { + ts, ok := v.(int64) + if !ok { + return false + } + now := time.Now().Unix() + + return now-ts < 10 +} diff --git a/database/schedule/update_test.go b/database/schedule/update_test.go index 2554cefdc..07cbfbc58 100644 --- a/database/schedule/update_test.go +++ b/database/schedule/update_test.go @@ -6,7 +6,6 @@ package schedule import ( "testing" - "time" "github.com/DATA-DOG/go-sqlmock" ) @@ -35,7 +34,7 @@ func TestSchedule_Engine_UpdateSchedule_Config(t *testing.T) { _mock.ExpectExec(`UPDATE "schedules" SET "repo_id"=$1,"active"=$2,"name"=$3,"entry"=$4,"created_at"=$5,"created_by"=$6,"updated_at"=$7,"updated_by"=$8,"scheduled_at"=$9 WHERE "id" = $10`). - WithArgs(1, false, "nightly", "0 0 * * *", 1, "user1", time.Now().UTC().Unix(), "user2", nil, 1). + WithArgs(1, false, "nightly", "0 0 * * *", 1, "user1", NowTimestamp{}, "user2", nil, 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/secret/secret_test.go b/database/secret/secret_test.go index 228567344..0d6a61c25 100644 --- a/database/secret/secret_test.go +++ b/database/secret/secret_test.go @@ -8,6 +8,7 @@ import ( "database/sql/driver" "reflect" "testing" + "time" "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/types/library" @@ -238,3 +239,17 @@ type AnyArgument struct{} func (a AnyArgument) Match(_ driver.Value) bool { return true } + +// NowTimestamp is used to test whether timestamps get updated correctly to the current time with lenience. +type NowTimestamp struct{} + +// Match satisfies sqlmock.Argument interface. +func (t NowTimestamp) Match(v driver.Value) bool { + ts, ok := v.(int64) + if !ok { + return false + } + now := time.Now().Unix() + + return now-ts < 10 +} diff --git a/database/secret/update_test.go b/database/secret/update_test.go index e719e220e..fd9a24ff6 100644 --- a/database/secret/update_test.go +++ b/database/secret/update_test.go @@ -6,7 +6,6 @@ package secret import ( "testing" - "time" "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/types/library" @@ -57,21 +56,21 @@ func TestSecret_Engine_UpdateSecret(t *testing.T) { _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"events"=$8,"allow_command"=$9,"created_at"=$10,"created_by"=$11,"updated_at"=$12,"updated_by"=$13 WHERE "id" = $14`). - WithArgs("foo", "bar", nil, "baz", AnyArgument{}, "repo", nil, nil, false, 1, "user", time.Now().UTC().Unix(), "user2", 1). + WithArgs("foo", "bar", nil, "baz", AnyArgument{}, "repo", nil, nil, false, 1, "user", AnyArgument{}, "user2", 1). WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the org query _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"events"=$8,"allow_command"=$9,"created_at"=$10,"created_by"=$11,"updated_at"=$12,"updated_by"=$13 WHERE "id" = $14`). - WithArgs("foo", "*", nil, "bar", AnyArgument{}, "org", nil, nil, false, 1, "user", time.Now().UTC().Unix(), "user2", 2). + WithArgs("foo", "*", nil, "bar", AnyArgument{}, "org", nil, nil, false, 1, "user", AnyArgument{}, "user2", 2). WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the shared query _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"events"=$8,"allow_command"=$9,"created_at"=$10,"created_by"=$11,"updated_at"=$12,"updated_by"=$13 WHERE "id" = $14`). - WithArgs("foo", nil, "bar", "baz", AnyArgument{}, "shared", nil, nil, false, 1, "user", time.Now().UTC().Unix(), "user2", 3). + WithArgs("foo", nil, "bar", "baz", AnyArgument{}, "shared", nil, nil, false, 1, "user", NowTimestamp{}, "user2", 3). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/service/clean_test.go b/database/service/clean_test.go index f30939880..3bd9baf15 100644 --- a/database/service/clean_test.go +++ b/database/service/clean_test.go @@ -7,7 +7,6 @@ package service import ( "reflect" "testing" - "time" "github.com/DATA-DOG/go-sqlmock" ) @@ -59,7 +58,7 @@ func TestService_Engine_CleanService(t *testing.T) { // ensure the mock expects the name query _mock.ExpectExec(`UPDATE "services" SET "status"=$1,"error"=$2,"finished"=$3 WHERE created < $4 AND (status = 'running' OR status = 'pending')`). - WithArgs("error", "msg", time.Now().UTC().Unix(), 3). + WithArgs("error", "msg", NowTimestamp{}, 3). WillReturnResult(sqlmock.NewResult(1, 2)) _sqlite := testSqlite(t) diff --git a/database/service/service_test.go b/database/service/service_test.go index 7749f43d3..c9d658769 100644 --- a/database/service/service_test.go +++ b/database/service/service_test.go @@ -5,8 +5,10 @@ package service import ( + "database/sql/driver" "reflect" "testing" + "time" "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/types/library" @@ -222,3 +224,23 @@ func testService() *library.Service { Distribution: new(string), } } + +// This will be used with the github.com/DATA-DOG/go-sqlmock library to compare values +// that are otherwise not easily compared. These typically would be values generated +// before adding or updating them in the database. +// +// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime + +// NowTimestamp is used to test whether timestamps get updated correctly to the current time with lenience. +type NowTimestamp struct{} + +// Match satisfies sqlmock.Argument interface. +func (t NowTimestamp) Match(v driver.Value) bool { + ts, ok := v.(int64) + if !ok { + return false + } + now := time.Now().Unix() + + return now-ts < 10 +} diff --git a/database/step/clean_test.go b/database/step/clean_test.go index 22709749a..4d0c68e07 100644 --- a/database/step/clean_test.go +++ b/database/step/clean_test.go @@ -7,7 +7,6 @@ package step import ( "reflect" "testing" - "time" "github.com/DATA-DOG/go-sqlmock" ) @@ -59,7 +58,7 @@ func TestStep_Engine_CleanStep(t *testing.T) { // ensure the mock expects the name query _mock.ExpectExec(`UPDATE "steps" SET "status"=$1,"error"=$2,"finished"=$3 WHERE created < $4 AND (status = 'running' OR status = 'pending')`). - WithArgs("error", "msg", time.Now().UTC().Unix(), 3). + WithArgs("error", "msg", NowTimestamp{}, 3). WillReturnResult(sqlmock.NewResult(1, 2)) _sqlite := testSqlite(t) diff --git a/database/step/step_test.go b/database/step/step_test.go index 26edfa5e1..136d5ca40 100644 --- a/database/step/step_test.go +++ b/database/step/step_test.go @@ -5,8 +5,10 @@ package step import ( + "database/sql/driver" "reflect" "testing" + "time" "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/types/library" @@ -223,3 +225,23 @@ func testStep() *library.Step { Distribution: new(string), } } + +// This will be used with the github.com/DATA-DOG/go-sqlmock library to compare values +// that are otherwise not easily compared. These typically would be values generated +// before adding or updating them in the database. +// +// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime + +// NowTimestamp is used to test whether timestamps get updated correctly to the current time with lenience. +type NowTimestamp struct{} + +// Match satisfies sqlmock.Argument interface. +func (t NowTimestamp) Match(v driver.Value) bool { + ts, ok := v.(int64) + if !ok { + return false + } + now := time.Now().Unix() + + return now-ts < 10 +} From 8a25e311b691aeba64fdb09608cd54c3872a0da8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 11:37:03 -0500 Subject: [PATCH 7/7] fix(deps): update deps (patch) (#895) * fix(deps): update deps (patch) * fix(deps): update deps (patch) --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: ecrupper --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index dea6f6204..48eba88e8 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 github.com/adhocore/gronx v1.6.3 - github.com/alicebob/miniredis/v2 v2.30.3 - github.com/aws/aws-sdk-go v1.44.281 + github.com/alicebob/miniredis/v2 v2.30.4 + github.com/aws/aws-sdk-go v1.44.298 github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.1 @@ -30,7 +30,7 @@ require ( github.com/redis/go-redis/v9 v9.0.5 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.9.5 - github.com/urfave/cli/v2 v2.25.6 + github.com/urfave/cli/v2 v2.25.7 go.starlark.net v0.0.0-20230612165344-9532f5667272 golang.org/x/oauth2 v0.9.0 golang.org/x/sync v0.3.0 @@ -38,7 +38,7 @@ require ( gorm.io/driver/postgres v1.5.2 gorm.io/driver/sqlite v1.5.2 gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 - k8s.io/apimachinery v0.27.2 + k8s.io/apimachinery v0.27.3 ) require ( diff --git a/go.sum b/go.sum index 2465b7cca..f1d227814 100644 --- a/go.sum +++ b/go.sum @@ -63,11 +63,11 @@ github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGn github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis/v2 v2.11.1/go.mod h1:UA48pmi7aSazcGAvcdKcBB49z521IC9VjTTRz2nIaJE= -github.com/alicebob/miniredis/v2 v2.30.3 h1:hrqDB4cHFSHQf4gO3xu6YKQg8PqJpNjLYsQAFYHstqw= -github.com/alicebob/miniredis/v2 v2.30.3/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg= +github.com/alicebob/miniredis/v2 v2.30.4 h1:8S4/o1/KoUArAGbGwPxcwf0krlzceva2XVOSchFS7Eo= +github.com/alicebob/miniredis/v2 v2.30.4/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.44.281 h1:z/ptheJvINaIAsKXthxONM+toTKw2pxyk700Hfm6yUw= -github.com/aws/aws-sdk-go v1.44.281/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.298 h1:5qTxdubgV7PptZJmp/2qDwD2JL187ePL7VOxsSh1i3g= +github.com/aws/aws-sdk-go v1.44.298/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -381,8 +381,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/urfave/cli/v2 v2.25.6 h1:yuSkgDSZfH3L1CjF2/5fNNg2KbM47pY2EvjBq4ESQnU= -github.com/urfave/cli/v2 v2.25.6/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -755,8 +755,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/apimachinery v0.27.2 h1:vBjGaKKieaIreI+oQwELalVG4d8f3YAMNpWLzDXkxeg= -k8s.io/apimachinery v0.27.2/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= +k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM= +k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY=