From f32f656a1d422a809deae255f0a06b73b882a099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20SZKIBA?= Date: Thu, 29 Aug 2024 12:13:56 +0200 Subject: [PATCH] feat: added support for running archive --- cmd/archive.go | 140 +++++++++++++++++++++++++++++++++++++ cmd/archive_test.go | 29 ++++++++ cmd/state.go | 32 ++++++--- cmd/state_internal_test.go | 3 + cmd/testdata/archive.tar | Bin 0 -> 11776 bytes cmd/testdata/combined.js | 13 ++++ cmd/testdata/faker.js | 11 +++ cmd/testdata/sqlite.js | 24 +++++++ examples/faker.js | 1 + go.mod | 2 +- releases/v0.1.6.md | 15 ++++ 11 files changed, 259 insertions(+), 11 deletions(-) create mode 100644 cmd/archive.go create mode 100644 cmd/archive_test.go create mode 100644 cmd/testdata/archive.tar create mode 100644 cmd/testdata/combined.js create mode 100644 cmd/testdata/faker.js create mode 100644 cmd/testdata/sqlite.js create mode 100644 releases/v0.1.6.md diff --git a/cmd/archive.go b/cmd/archive.go new file mode 100644 index 0000000..5d44f04 --- /dev/null +++ b/cmd/archive.go @@ -0,0 +1,140 @@ +package cmd + +import ( + "archive/tar" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strings" + + "github.com/grafana/k6deps" + "github.com/grafana/k6pack" +) + +//nolint:forbidigo +func analyzeArchive(filename string) (k6deps.Dependencies, error) { + dir, err := os.MkdirTemp("", "k6-archive-*") + if err != nil { + return nil, err + } + + defer os.RemoveAll(dir) //nolint:errcheck + + err = extractArchive(dir, filename) + if err != nil { + return nil, err + } + + opts, err := loadMetadata(dir) + if err != nil { + return nil, err + } + + return k6deps.Analyze(opts) +} + +//nolint:forbidigo +func loadMetadata(dir string) (*k6deps.Options, error) { + var meta archiveMetadata + + data, err := os.ReadFile(filepath.Join(filepath.Clean(dir), "metadata.json")) + if err != nil { + return nil, err + } + + if err = json.Unmarshal(data, &meta); err != nil { + return nil, err + } + + opts := new(k6deps.Options) + + opts.Manifest.Ignore = true // no manifest (yet) in archive + + opts.Script.Name = filepath.Join( + dir, + "file", + filepath.FromSlash(strings.TrimPrefix(meta.Filename, "file:///")), + ) + + if value, found := meta.Env[k6deps.EnvDependencies]; found { + opts.Env.Name = k6deps.EnvDependencies + opts.Env.Contents = []byte(value) + } else { + opts.Env.Ignore = true + } + + contents, err := os.ReadFile(filepath.Join(filepath.Clean(dir), "data")) + if err != nil { + return nil, err + } + + script, _, err := k6pack.Pack(string(contents), &k6pack.Options{Filename: opts.Script.Name}) + if err != nil { + return nil, err + } + + opts.Script.Contents = script + + return opts, nil +} + +type archiveMetadata struct { + Filename string `json:"filename"` + Env map[string]string `json:"env"` +} + +//nolint:forbidigo +func extractArchive(dir string, filename string) error { + input, err := os.Open(filepath.Clean(filename)) + if err != nil { + return err + } + + defer input.Close() //nolint:errcheck + + reader := tar.NewReader(input) + + const maxFileSize = 1024 * 1024 * 10 // 10M + + for { + header, err := reader.Next() + + switch { + case err == io.EOF: + return nil + case err != nil: + return err + case header == nil: + continue + } + + target := filepath.Join(dir, filepath.Clean(filepath.FromSlash(header.Name))) + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0o750); err != nil { + return err + } + + case tar.TypeReg: + if ext := filepath.Ext(target); ext == ".csv" || (ext == ".json" && filepath.Base(target) != "metadata.json") { + continue + } + + file, err := os.OpenFile(filepath.Clean(target), os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return err + } + + if _, err := io.CopyN(file, reader, maxFileSize); err != nil && !errors.Is(err, io.EOF) { + return err + } + + if err = file.Close(); err != nil { + return err + } + } + } +} diff --git a/cmd/archive_test.go b/cmd/archive_test.go new file mode 100644 index 0000000..1217b0f --- /dev/null +++ b/cmd/archive_test.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "path/filepath" + "testing" + + "github.com/grafana/k6deps" + "github.com/stretchr/testify/require" +) + +func Test_analyzeArchive(t *testing.T) { + t.Parallel() + + actual, err := analyzeArchive(filepath.Join("testdata", "archive.tar")) + + require.NoError(t, err) + + opts := &k6deps.Options{ + Script: k6deps.Source{Name: filepath.Join("testdata", "combined.js")}, + Manifest: k6deps.Source{Ignore: true}, + Env: k6deps.Source{Ignore: true}, + } + + expected, err := k6deps.Analyze(opts) + + require.NoError(t, err) + + require.Equal(t, expected, actual) +} diff --git a/cmd/state.go b/cmd/state.go index 08db724..eb6044f 100644 --- a/cmd/state.go +++ b/cmd/state.go @@ -6,6 +6,7 @@ import ( "net/url" "os" "os/exec" + "strings" "github.com/grafana/k6deps" "github.com/grafana/k6exec" @@ -68,18 +69,29 @@ func (s *state) persistentPreRunE(_ *cobra.Command, _ []string) error { return nil } -func (s *state) preRunE(sub *cobra.Command, args []string) error { - var ( - deps k6deps.Dependencies - err error - dopts k6deps.Options - ) - - if scriptname, hasScript := scriptArg(sub, args); hasScript { - dopts.Script.Name = scriptname +func analyze(sub *cobra.Command, args []string) (k6deps.Dependencies, error) { + scriptname, hasScript := scriptArg(sub, args) + if !hasScript { + return k6deps.Analyze(&k6deps.Options{}) + } + + if strings.HasSuffix(scriptname, ".tar") { + return analyzeArchive(scriptname) } - deps, err = k6deps.Analyze(&dopts) + return analyzeScript(scriptname) +} + +func analyzeScript(filename string) (k6deps.Dependencies, error) { + var opts k6deps.Options + + opts.Script.Name = filename + + return k6deps.Analyze(&opts) +} + +func (s *state) preRunE(sub *cobra.Command, args []string) error { + deps, err := analyze(sub, args) if err != nil { return err } diff --git a/cmd/state_internal_test.go b/cmd/state_internal_test.go index 28f7f18..a3eb9d7 100644 --- a/cmd/state_internal_test.go +++ b/cmd/state_internal_test.go @@ -113,6 +113,9 @@ func Test_preRunE(t *testing.T) { arg := filepath.Join("testdata", "script.js") require.NoError(t, st.preRunE(sub, []string{arg})) + arg = filepath.Join("testdata", "archive.tar") + require.NoError(t, st.preRunE(sub, []string{arg})) + arg = filepath.Join("testdata", "invalid_constraint.js") require.Error(t, st.preRunE(sub, []string{arg})) diff --git a/cmd/testdata/archive.tar b/cmd/testdata/archive.tar new file mode 100644 index 0000000000000000000000000000000000000000..485add9cab4a060cb364589e11f13106857e3264 GIT binary patch literal 11776 zcmeHNZFAZ<5au(#LgVR7;3mbqGZ&_vp(&kmmyiyS-gRznQojgLu`O4Y2{e=cek<7q z+oS|1^!lOp7tm_8TJ57<%e%5M*G!FTl`kH33P0wl!qaXx3*r}^^lziq?i6axcDq?^ z)@!YHp<1hT8tnqA{up(VQhkqW3{*-|v$uk^Z5Nka!cQ4-S42qRYZohLNcrL^r=m?= zPN?GnNpu6j)qD?|&~|*w5-dPl`hp=wFny+paY-uSI3TCE!R#}frWx4L7@;XLUSY!* zlE-*vV}~avNcWEMS06iuSWrwkhmJA#-6YWyqzSPx^;1F}I>8z<>1}E?JfuE2h`A}n z&6o{^nYopiA4ndyu#ucwn44O}SckmQ+V`Q+ZEZEeCSforH5UNHdb1tt=o&ZXiCGm% z`heMc>Lq7Z=A654vF^{31Bl}R01S^8#GPD>KVn9vk#?b8-15$eJI8G71K=supcBS; zJ6i+jJ)ftv<8wxDlk)=!dvj`;DY?3(84HVeeDGnXhZ+m9K1Z|D+RTBRVt42{bsVrM zs5ru4$>jVF{dm2X!6wci+U9Q`^a8v}ncDMhTVv}9!;U%Tn$+kY2{quaEoagZrJsTD zSOdH;FgdWnX4p2|((l!BqS-1dQsBQKM$we8->)5#)@SW;Y)`bAFwI60V~&l*Rw!nS zvQ+cNpeBydRxa2Pn>PTN3o#_tHDeSUaH~D$a~fNXfNI<8GRo<;>4lLYM!{@Jz{@xC zAfRvvN!zxsr-Z%)@;|p-Wz{9g>6p0?QD{BN*4rs zBMf0Bxq(dL84yjNN1EgfHGwZ=0f?k?D^p@Y^aC3}AQu16R8^f*8>{)1$X1(4Xzs0)`^2nc$G^lDDJHY*-sJ{2*Az~1l$Gsfp{0+2P$_o&{t1<^HiwP?STu&cU7b#*Bd!75$_6*1KS8&W&qfQ_+9A#ZZZVLJE@ zy_2crD1*DrE<;MVXh5|vN*TXG4!#YzOSKy0Yol~N;AO=$lS1dX! z1sl|bLLYT1Q^GtxgtA(xEYiNkA}koH{jSf?&>z$z+Wgl}1ZR)`n=Jhy-X(b+@&8)A zQ;Yn+)@e!qZ+0O5=l*}NiJ#24-~VS!n`%xF008%m`$L&$>;A!pXxxZGv~d5>+kqTb zYNVe-bL-`%4h1KmRH%!ck`hW#PDPnWiZOL1=JgwIm$jhKQ4d0s zj7%Z-_d;B>US%*G_eT>n7*4L@3j`fcK?7KA8GY= 0.3.0"; +"use k6 with k6/x/sql >= 0.4.0"; + +import faker from "./faker.js"; +import sqlite from "./sqlite.js"; + +export { setup, teardown } from "./sqlite.js"; + +export default () => { + faker(); + sqlite(); +}; diff --git a/cmd/testdata/faker.js b/cmd/testdata/faker.js new file mode 100644 index 0000000..cf10c85 --- /dev/null +++ b/cmd/testdata/faker.js @@ -0,0 +1,11 @@ +// source: https://github.com/szkiba/xk6-faker/blob/v0.3.0/examples/custom-faker.js +"use k6 = 0.52"; +import { Faker } from "k6/x/faker"; + +const faker = new Faker(11); + +export default function () { + console.log(faker.person.firstName()); +} + +// output: Josiah diff --git a/cmd/testdata/sqlite.js b/cmd/testdata/sqlite.js new file mode 100644 index 0000000..c17f93e --- /dev/null +++ b/cmd/testdata/sqlite.js @@ -0,0 +1,24 @@ +// source: https://github.com/grafana/xk6-sql/blob/v0.4.0/examples/sqlite3_test.js +import sql from "k6/x/sql"; + +const db = sql.open("sqlite3", "./test.db"); + +export function setup() { + db.exec(`CREATE TABLE IF NOT EXISTS keyvalues ( + id integer PRIMARY KEY AUTOINCREMENT, + key varchar NOT NULL, + value varchar);`); +} + +export function teardown() { + db.close(); +} + +export default function () { + db.exec("INSERT INTO keyvalues (key, value) VALUES('plugin-name', 'k6-plugin-sql');"); + + let results = sql.query(db, "SELECT * FROM keyvalues WHERE key = $1;", "plugin-name"); + for (const row of results) { + console.log(`key: ${row.key}, value: ${row.value}`); + } +} diff --git a/examples/faker.js b/examples/faker.js index 4f38927..cf10c85 100644 --- a/examples/faker.js +++ b/examples/faker.js @@ -1,4 +1,5 @@ // source: https://github.com/szkiba/xk6-faker/blob/v0.3.0/examples/custom-faker.js +"use k6 = 0.52"; import { Faker } from "k6/x/faker"; const faker = new Faker(11); diff --git a/go.mod b/go.mod index ccf0e6e..dd429e5 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/grafana/clireadme v0.1.0 github.com/grafana/k6build v0.3.0 github.com/grafana/k6deps v0.1.4 + github.com/grafana/k6pack v0.2.2 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 github.com/samber/slog-logrus/v2 v2.5.0 github.com/sirupsen/logrus v1.9.3 @@ -23,7 +24,6 @@ require ( github.com/google/btree v1.1.2 // indirect github.com/grafana/k6catalog v0.1.0 // indirect github.com/grafana/k6foundry v0.2.0 // indirect - github.com/grafana/k6pack v0.2.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/releases/v0.1.6.md b/releases/v0.1.6.md new file mode 100644 index 0000000..8e668df --- /dev/null +++ b/releases/v0.1.6.md @@ -0,0 +1,15 @@ +k6exec `v0.1.6` is here 🎉! + +This is an internal maintenance release. + +**New features**: + +- The archive file created with the `archive` command can now be run using the `run` command. + ```bash + k6exec run archive.tar + ``` + +**Dependency upgrades**: + +- k6build v0.3.0 +