diff --git a/pkg/hookt/engine.go b/pkg/hookt/engine.go index 3c8b369..cd080b0 100644 --- a/pkg/hookt/engine.go +++ b/pkg/hookt/engine.go @@ -11,6 +11,7 @@ import ( "hookt.dev/cmd/pkg/errors" "hookt.dev/cmd/pkg/plugin/builtin" "hookt.dev/cmd/pkg/proto" + "hookt.dev/cmd/pkg/trace" ) var plugins []proto.Interface @@ -53,7 +54,9 @@ func (e *Engine) Run(ctx context.Context, file string) (*check.S, error) { var g errgroup.Group for _, job := range w.Jobs { + ctx := trace.With(ctx, "job", job.ID) for _, step := range job.Steps { + ctx := trace.With(ctx, "step", step.ID) g.Go(func() error { r, ok := step.With.(proto.Runner) if !ok { diff --git a/pkg/id/id.go b/pkg/id/id.go new file mode 100644 index 0000000..d6fb252 --- /dev/null +++ b/pkg/id/id.go @@ -0,0 +1,35 @@ +package id + +import ( + "math/rand" + "strings" +) + +const ( + all = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" +) + +func Gen(n int) string { + return gen(all, 6, 1<<6-1, 63/6, n) +} + +// source: https://stackoverflow.com/questions/22892120/ +func gen(chars string, idxBits uint, idxMask int64, idxMax int, n int) string { + var sb strings.Builder + + sb.Grow(n) + + for i, cache, remain := n-1, rand.Int63(), idxMax; i >= 0; { + if remain == 0 { + cache, remain = rand.Int63(), idxMax + } + if idx := int(cache & idxMask); idx < len(chars) { + sb.WriteByte(chars[idx]) + i-- + } + cache >>= idxBits + remain-- + } + + return sb.String() +} diff --git a/pkg/plugin/builtin/event/event.go b/pkg/plugin/builtin/event/event.go index 06674ff..66a1b65 100644 --- a/pkg/plugin/builtin/event/event.go +++ b/pkg/plugin/builtin/event/event.go @@ -2,7 +2,6 @@ package event // import "hookt.dev/cmd/pkg/plugin/builtin/event" import ( "context" - "encoding/json" "log/slog" "time" @@ -94,7 +93,7 @@ func (p *Plugin) process() { } } -func (p *Plugin) Step(context.Context) any { +func (p *Plugin) Step(ctx context.Context) any { c := make(chan proto.Message) p.c[c] = struct{}{} it, _ := time.ParseDuration(p.Config.InactiveTimeout) @@ -113,32 +112,26 @@ type Step struct { it time.Duration } -func str(v any) string { - if v == nil { - return "" - } - p, _ := json.Marshal(v) - return string(p) -} - func (s *Step) Run(ctx context.Context, c *check.S) error { slog.Debug("event: run", - "match", str(s.Match), - "pass", str(s.Pass), - "fail", str(s.Fail), + "match", s.Match, + "pass", s.Pass, + "fail", s.Fail, ) - match, err := s.p.p.Pattern(ctx, s.Match) + // tr := trace.ContextStep(ctx) + + match, err := s.p.p.Patterns(ctx, s.Match) if err != nil { return errors.New("failed to parse match pattern: %w", err) } - pass, err := s.p.p.Pattern(ctx, s.Pass) + pass, err := s.p.p.Patterns(ctx, s.Pass) if err != nil { return errors.New("failed to parse pass pattern: %w", err) } - fail, err := s.p.p.Pattern(ctx, s.Fail) + fail, err := s.p.p.Patterns(ctx, s.Fail) if err != nil { return errors.New("failed to parse fail pattern: %w", err) } diff --git a/pkg/proto/pattern.go b/pkg/proto/pattern.go index 5080711..0518946 100644 --- a/pkg/proto/pattern.go +++ b/pkg/proto/pattern.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "sort" "text/template" "hookt.dev/cmd/pkg/errors" @@ -15,14 +16,19 @@ import ( "sigs.k8s.io/yaml" ) -type Pattern map[*gojq.Query]func(context.Context, any) (bool, error) +type Pattern struct { + Key *gojq.Query + Match func(context.Context, any) (bool, error) +} + +type Patterns []*Pattern -func (p Pattern) Match(ctx context.Context, obj any) (bool, error) { - for q, fn := range p { - it := q.RunWithContext(ctx, obj) +func (p Patterns) Match(ctx context.Context, obj any) (bool, error) { + for _, p := range p { + it := p.Key.RunWithContext(ctx, obj) slog.Debug("pattern", - "query", q.String(), + "query", p.Key.String(), ) // TODO: Handle multiple results? @@ -31,9 +37,9 @@ func (p Pattern) Match(ctx context.Context, obj any) (bool, error) { return false, nil } - ok, err := fn(ctx, v) + ok, err := p.Match(ctx, v) if err != nil { - return false, errors.New("failed to match jq %q: %w", q.String(), err) + return false, errors.New("failed to match jq %q: %w", p.Key.String(), err) } if !ok { return false, nil @@ -43,16 +49,21 @@ func (p Pattern) Match(ctx context.Context, obj any) (bool, error) { return len(p) != 0, nil } -func (p *P) Pattern(ctx context.Context, obj wire.Object) (Pattern, error) { +func (p *P) Patterns(ctx context.Context, obj wire.Object) (Patterns, error) { var ( - pt = make(Pattern) + pt = make(Patterns, 0, len(obj)) tr = trace.ContextPattern(ctx) err error ) for k, raw := range obj { - q, e := gojq.Parse(k) - tr.ParseKey(k, q, e) + var ( + e error + q Pattern + ) + + q.Key, e = gojq.Parse(k) + tr.ParseKey(k, q.Key, e) if e != nil { err = errors.Join( err, @@ -80,20 +91,26 @@ func (p *P) Pattern(ctx context.Context, obj wire.Object) (Pattern, error) { switch want := want.(type) { case bool: - pt[q] = func(_ context.Context, got any) (bool, error) { return want || got == nil, nil } + q.Match = func(_ context.Context, got any) (bool, error) { return want || got == nil, nil } case string: tv := tr.TemplateValue tr.TemplateValue = func(_, v string, t *template.Template, e error) { tv(k, v, t, e) } - pt[q] = p.t.Match(trace.WithPattern(ctx, tr), want) + q.Match = p.t.Match(trace.WithPattern(ctx, tr), want) default: - pt[q] = func(_ context.Context, got any) (bool, error) { + q.Match = func(_ context.Context, got any) (bool, error) { ok := cmpEqual(want, got) tr.EqualMatch(k, want, got, ok) return ok, nil } } + + pt = append(pt, &q) } + sort.Slice(pt, func(i, j int) bool { + return pt[i].Key.String() < pt[j].Key.String() + }) + return pt, err } diff --git a/pkg/proto/proto.go b/pkg/proto/proto.go index 54cd65b..5493920 100644 --- a/pkg/proto/proto.go +++ b/pkg/proto/proto.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "log/slog" + "strconv" + "strings" "hookt.dev/cmd/pkg/errors" "hookt.dev/cmd/pkg/proto/wire" @@ -17,6 +19,7 @@ type Workflow struct { } type Job struct { + ID string Plugins []Plugin Steps []Step } @@ -66,9 +69,15 @@ func (p *P) Parse(ctx context.Context, q []byte) (*Workflow, error) { w.Jobs = make([]Job, len(raw.Jobs)) + uniq := make(map[string]struct{}) + for i, job := range raw.Jobs { j := &w.Jobs[i] + if strings.HasPrefix(job.ID, "#") { + return nil, errors.New("#job-%d: error reading job: id cannot start with #", i) + } + slog.Debug("wiring jobs", "index", i, "job", job.ID, @@ -76,8 +85,17 @@ func (p *P) Parse(ctx context.Context, q []byte) (*Workflow, error) { tr.WireJob(i, &job) + j.ID = nonempty(job.ID, "#job-"+strconv.Itoa(i)) j.Plugins = make([]Plugin, len(job.Plugins)) + if _, ok := uniq[j.ID]; ok { + return nil, errors.New("#job-%d: error reading job: duplicate id %q", i, j.ID) + } + + uniq[j.ID] = struct{}{} + + ctx := trace.With(ctx, "job", j.ID) + for k, plugin := range job.Plugins { iface, ok := p.m[plugin.Uses] if !ok { @@ -104,29 +122,41 @@ func (p *P) Parse(ctx context.Context, q []byte) (*Workflow, error) { j.Steps = make([]Step, len(job.Steps)) + uniq := make(map[string]struct{}) + for k, step := range job.Steps { iface, ok := p.m[step.Uses] if !ok { - return nil, errors.New("error reading plugin %q step: not found", step.Uses) + return nil, errors.New("#step-%d: error reading plugin %q step: not found", k, step.Uses) } - slog.Debug("wiring steps", - "index", k, - "step", step.Uses, - "with", string(step.With), - ) + if strings.HasPrefix(step.ID, "#") { + return nil, errors.New("#step-%d: error reading plugin %q step: id cannot start with #", k, step.Uses) + } s := &j.Steps[k] s.Uses = step.Uses - s.ID = step.ID + s.ID = nonempty(step.ID, "#step-"+strconv.Itoa(k)) s.Desc = step.Desc - s.With = iface.Step(ctx) + s.With = iface.Step(trace.With(ctx, "step", s.ID)) + + if _, ok := uniq[s.ID]; ok { + return nil, errors.New("error reading plugin %q step: duplicate id %q", step.Uses, s.ID) + } + + uniq[s.ID] = struct{}{} if err := yaml.Unmarshal(step.With, s.With); err != nil { - return nil, errors.New("error reading plugin %q step: %w", step.Uses, err) + return nil, errors.New("%s: error reading plugin %q step: %w", s.ID, step.Uses, err) } + slog.Debug("wiring steps", + "id", s.ID, + "step", step.Uses, + "with", string(step.With), + ) + tr.WireStep(k, &step, s.With) } @@ -151,3 +181,13 @@ func (p *P) Parse(ctx context.Context, q []byte) (*Workflow, error) { return &w, nil } + +func nonempty[T comparable](t ...T) T { + var zero T + for _, v := range t { + if v != zero { + return v + } + } + return zero +} diff --git a/pkg/proto/template.go b/pkg/proto/template.go index 02b727e..ef01164 100644 --- a/pkg/proto/template.go +++ b/pkg/proto/template.go @@ -64,20 +64,21 @@ func (t *T) funcs() template.FuncMap { } func (t *T) Match(ctx context.Context, data string) func(context.Context, any) (bool, error) { + const inherit = "" // the key is going to be overriden + tr := trace.ContextPattern(ctx) tmpl, err := t.Parse("", data) - tr.TemplateValue("", data, tmpl, err) + tr.TemplateValue(inherit, data, tmpl, err) if err != nil { return func(context.Context, any) (bool, error) { return false, err } } return func(ctx context.Context, got any) (bool, error) { - var ( - buf bytes.Buffer - tr = trace.ContextPattern(ctx) - ) + var buf bytes.Buffer + // TODO: Fix + // tr = trace.ContextPattern(ctx) err := tmpl.Execute(&buf, got) - tr.ExecuteMatch("", buf.Bytes(), err) + tr.ExecuteMatch(inherit, buf.Bytes(), err) if err != nil { return false, errors.New("failed to evaluate %q: %w", data, err) } @@ -85,7 +86,7 @@ func (t *T) Match(ctx context.Context, data string) func(context.Context, any) ( var want any err = yaml.Unmarshal(buf.Bytes(), &want) - tr.UnmarshalMatch("", buf.Bytes(), want, err) + tr.UnmarshalMatch(inherit, buf.Bytes(), want, err) if err != nil { return false, errors.New("failed to parse result: %w", err) } @@ -95,7 +96,7 @@ func (t *T) Match(ctx context.Context, data string) func(context.Context, any) ( return want, nil default: ok := cmpEqual(want, got) - tr.EqualMatch("", want, got, ok) + tr.EqualMatch(inherit, want, got, ok) return ok, nil } } diff --git a/pkg/proto/wire/model.go b/pkg/proto/wire/model.go index 62f11f7..06152df 100644 --- a/pkg/proto/wire/model.go +++ b/pkg/proto/wire/model.go @@ -2,6 +2,7 @@ package wire import ( "encoding/json" + "log/slog" ) type Workflow struct { @@ -9,7 +10,7 @@ type Workflow struct { } type Job struct { - ID string `json:"id"` + ID string `json:"id,omitempty"` Plugins []Plugin `json:"plugins"` Steps []Step `json:"steps"` } @@ -36,6 +37,14 @@ type Generic = json.RawMessage type Object map[string]Generic +func (o Object) LogValue() slog.Value { + p, err := json.Marshal(o) + if err != nil { + panic("unexpected error: " + err.Error()) + } + return slog.StringValue(string(p)) +} + type Step struct { Uses string `json:"uses"` ID string `json:"id,omitempty"` diff --git a/pkg/trace/trace.go b/pkg/trace/trace.go index f900eea..52a9cbf 100644 --- a/pkg/trace/trace.go +++ b/pkg/trace/trace.go @@ -20,6 +20,13 @@ var ( RunStep: func() {}, MatchStep: func() {}, TapMessage: func() {}, + Step: func(string) StepTrace { return nopStep }, + } + nopStep = StepTrace{ + PatternGroup: func(string, string) PatternGroupTrace { return nopPatternGroup }, + } + nopPatternGroup = PatternGroupTrace{ + Pattern: func(string) PatternTrace { return nopPattern }, } nopPattern = PatternTrace{ ParseKey: func(string, *gojq.Query, error) {}, @@ -145,6 +152,28 @@ func ContextJob(ctx context.Context) JobTrace { return nopJob } +func WithStep(ctx context.Context, trace StepTrace) context.Context { + return with(ctx, &trace) +} + +func ContextStep(ctx context.Context) StepTrace { + if trace := from[StepTrace](ctx); trace != nil { + return *trace + } + return nopStep +} + +func WithPatternGroup(ctx context.Context, trace PatternGroupTrace) context.Context { + return with(ctx, &trace) +} + +func ContextPatternGroup(ctx context.Context) PatternGroupTrace { + if trace := from[PatternGroupTrace](ctx); trace != nil { + return *trace + } + return nopPatternGroup +} + func WithPattern(ctx context.Context, trace PatternTrace) context.Context { return with(ctx, &trace) } @@ -156,6 +185,26 @@ func ContextPattern(ctx context.Context) PatternTrace { return nopPattern } +type Bag map[string]string + +func With(ctx context.Context, key, value string) context.Context { + bag := from[Bag](ctx) + if bag == nil { + bag = &Bag{} + } + (*bag)[key] = value + return with(ctx, bag) +} + +func Get(ctx context.Context, key string) string { + if bag := from[Bag](ctx); bag != nil { + return (*bag)[key] + } + return "" +} + +type Trace struct{} + type JobTrace struct { WireJob func(index int, job *wire.Job) WirePlugin func(index int, plugin *wire.Plugin, impl any) @@ -164,6 +213,16 @@ type JobTrace struct { MatchStep func() TapMessage func() + + Step func(id string) StepTrace +} + +type StepTrace struct { + PatternGroup func(step, name string) PatternGroupTrace +} + +type PatternGroupTrace struct { + Pattern func(key string) PatternTrace } type PatternTrace struct {