Skip to content

Commit

Permalink
fix: Skip tables.json when packaging destinations (#1238)
Browse files Browse the repository at this point in the history 
This adds a new `type` (`source` or `destination`) argument to the `package` command, and switches the ordering of arguments bit (since we're making a breaking change to it anyway). The `type` is written to package.json as well, and if it is `destination`, we don't create a `tables.json`.
  • Loading branch information
@hermanschaaf
hermanschaaf authored Sep 21, 2023
1 parent f6c1cab commit f6471e3
Show file tree
Hide file tree
Showing 4 changed files with 131 additions and 15 deletions.
15 changes: 12 additions & 3 deletions examples/simple_plugin/plugin/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"

"github.com/apache/arrow/go/v14/arrow"
"github.com/cloudquery/plugin-sdk/examples/simple_plugin/client"
"github.com/cloudquery/plugin-sdk/examples/simple_plugin/services"
"github.com/cloudquery/plugin-sdk/v4/message"
Expand All @@ -20,8 +21,6 @@ type Client struct {
config client.Spec
tables schema.Tables
scheduler *scheduler.Scheduler

plugin.UnimplementedDestination
}

func (c *Client) Logger() *zerolog.Logger {
Expand Down Expand Up @@ -55,6 +54,16 @@ func (*Client) Close(_ context.Context) error {
return nil
}

func (*Client) Write(context.Context, <-chan message.WriteMessage) error {
// Not implemented, just used for testing destination packaging
return nil
}

func (*Client) Read(context.Context, *schema.Table, chan<- arrow.Record) error {
// Not implemented, just used for testing destination packaging
return nil
}

func Configure(_ context.Context, logger zerolog.Logger, spec []byte, opts plugin.NewClientOptions) (plugin.Client, error) {
if opts.NoConnection {
return &Client{
Expand Down Expand Up @@ -93,4 +102,4 @@ func getTables() schema.Tables {
schema.AddCqIDs(t)
}
return tables
}
}
18 changes: 18 additions & 0 deletions plugin/plugin_package.go
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package plugin

import "errors"

const (
GoOSLinux = "linux"
GoOSWindows = "windows"
Expand All @@ -9,6 +11,22 @@ const (
GoArchArm64 = "arm64"
)

type Type string

const (
TypeSource Type = "source"
TypeDestination Type = "destination"
)

func (t Type) Validate() error {
switch t {
case TypeSource, TypeDestination:
return nil
default:
return errors.New("invalid type: must be one of source, destination")
}
}

type PackageType string

const (
Expand Down
22 changes: 15 additions & 7 deletions serve/package.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ This creates a directory with the plugin binaries, package.json and documentatio
type PackageJSON struct {
SchemaVersion int `json:"schema_version"`
Name string `json:"name"`
Type plugin.Type `json:"type"`
Message string `json:"message"`
Version string `json:"version"`
Protocols []int `json:"protocols"`
Expand Down Expand Up @@ -199,11 +200,12 @@ func (*PluginServe) getModuleName(pluginDirectory string) (string, error) {
return strings.TrimSpace(importPath), nil
}

func (s *PluginServe) writePackageJSON(dir, pluginVersion, message string, targets []TargetBuild) error {
func (s *PluginServe) writePackageJSON(dir string, pluginType plugin.Type, pluginVersion, message string, targets []TargetBuild) error {
packageJSON := PackageJSON{
SchemaVersion: 1,
Name: s.plugin.Name(),
Message: message,
Type: pluginType,
Version: pluginVersion,
Protocols: s.versions,
SupportedTargets: targets,
Expand Down Expand Up @@ -266,13 +268,17 @@ func copyFile(src, dst string) error {

func (s *PluginServe) newCmdPluginPackage() *cobra.Command {
cmd := &cobra.Command{
Use: "package -m <message> <plugin_directory> <version>",
Use: "package -m <message> <source|destination> <version> <plugin_directory>",
Short: pluginPackageShort,
Long: pluginPackageLong,
Args: cobra.ExactArgs(2),
Args: cobra.ExactArgs(3),
RunE: func(cmd *cobra.Command, args []string) error {
pluginDirectory := args[0]
pluginType := plugin.Type(args[0])
if err := pluginType.Validate(); err != nil {
return err
}
pluginVersion := args[1]
pluginDirectory := args[2]
distPath := path.Join(pluginDirectory, "dist")
if cmd.Flag("dist-dir").Changed {
distPath = cmd.Flag("dist-dir").Value.String()
Expand Down Expand Up @@ -304,8 +310,10 @@ func (s *PluginServe) newCmdPluginPackage() *cobra.Command {
}); err != nil {
return err
}
if err := s.writeTablesJSON(cmd.Context(), distPath); err != nil {
return err
if pluginType == plugin.TypeSource {
if err := s.writeTablesJSON(cmd.Context(), distPath); err != nil {
return err
}
}
targets := []TargetBuild{}
for _, target := range s.plugin.Targets() {
Expand All @@ -316,7 +324,7 @@ func (s *PluginServe) newCmdPluginPackage() *cobra.Command {
}
targets = append(targets, *targetBuild)
}
if err := s.writePackageJSON(distPath, pluginVersion, message, targets); err != nil {
if err := s.writePackageJSON(distPath, pluginType, pluginVersion, message, targets); err != nil {
return fmt.Errorf("failed to write manifest: %w", err)
}
if err := s.copyDocs(distPath, docsPath); err != nil {
Expand Down
91 changes: 86 additions & 5 deletions serve/package_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
"github.com/google/go-cmp/cmp"
)

func TestPluginPackage(t *testing.T) {
func TestPluginPackage_Source(t *testing.T) {
_, filename, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("failed to get current file path")
Expand All @@ -30,7 +30,6 @@ func TestPluginPackage(t *testing.T) {
plugin.WithBuildTargets([]plugin.BuildTarget{
{OS: plugin.GoOSLinux, Arch: plugin.GoArchAmd64},
{OS: plugin.GoOSWindows, Arch: plugin.GoArchAmd64},
{OS: plugin.GoOSDarwin, Arch: plugin.GoArchAmd64},
}),
)
msg := `Test message
Expand All @@ -54,7 +53,7 @@ with multiple lines and **markdown**`
srv := Plugin(p)
cmd := srv.newCmdPluginRoot()
distDir := t.TempDir()
cmd.SetArgs([]string{"package", "--dist-dir", distDir, "-m", tc.message, simplePluginPath, packageVersion})
cmd.SetArgs([]string{"package", "--dist-dir", distDir, "-m", tc.message, "source", packageVersion, simplePluginPath})
err := cmd.Execute()
if tc.wantErr && err == nil {
t.Fatalf("expected error, got nil")
Expand All @@ -68,7 +67,6 @@ with multiple lines and **markdown**`
expect := []string{
"docs",
"package.json",
"plugin-testPlugin-v1.2.3-darwin-amd64.zip",
"plugin-testPlugin-v1.2.3-linux-amd64.zip",
"plugin-testPlugin-v1.2.3-windows-amd64.zip",
"tables.json",
Expand All @@ -80,13 +78,13 @@ with multiple lines and **markdown**`
expectPackage := PackageJSON{
SchemaVersion: 1,
Name: "testPlugin",
Type: "source",
Message: msg,
Version: "v1.2.3",
Protocols: []int{3},
SupportedTargets: []TargetBuild{
{OS: plugin.GoOSLinux, Arch: plugin.GoArchAmd64, Path: "plugin-testPlugin-v1.2.3-linux-amd64.zip", Checksum: "sha256:" + sha256sum(filepath.Join(distDir, "plugin-testPlugin-v1.2.3-linux-amd64.zip"))},
{OS: plugin.GoOSWindows, Arch: plugin.GoArchAmd64, Path: "plugin-testPlugin-v1.2.3-windows-amd64.zip", Checksum: "sha256:" + sha256sum(filepath.Join(distDir, "plugin-testPlugin-v1.2.3-windows-amd64.zip"))},
{OS: plugin.GoOSDarwin, Arch: plugin.GoArchAmd64, Path: "plugin-testPlugin-v1.2.3-darwin-amd64.zip", Checksum: "sha256:" + sha256sum(filepath.Join(distDir, "plugin-testPlugin-v1.2.3-darwin-amd64.zip"))},
},
PackageType: plugin.PackageTypeNative,
}
Expand All @@ -102,6 +100,89 @@ with multiple lines and **markdown**`
}
}

func TestPluginPackage_Destination(t *testing.T) {
_, filename, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("failed to get current file path")
}
dir := filepath.Dir(filepath.Dir(filename))
simplePluginPath := filepath.Join(dir, "examples/simple_plugin")
packageVersion := "v1.2.3"
p := plugin.NewPlugin(
"testPlugin",
"development",
memdb.NewMemDBClient,
plugin.WithBuildTargets([]plugin.BuildTarget{
{OS: plugin.GoOSWindows, Arch: plugin.GoArchAmd64},
{OS: plugin.GoOSDarwin, Arch: plugin.GoArchAmd64},
}),
)
msg := `Test message
with multiple lines and **markdown**`
testCases := []struct {
name string
message string
wantErr bool
}{
{
name: "inline message",
message: msg,
},
{
name: "message from file",
message: "@testdata/message.txt",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
srv := Plugin(p)
cmd := srv.newCmdPluginRoot()
distDir := t.TempDir()
cmd.SetArgs([]string{"package", "--dist-dir", distDir, "-m", tc.message, "destination", packageVersion, simplePluginPath})
err := cmd.Execute()
if tc.wantErr && err == nil {
t.Fatalf("expected error, got nil")
} else if err != nil {
t.Fatalf("unexpected error: %v", err)
}
files, err := os.ReadDir(distDir)
if err != nil {
t.Fatal(err)
}
expect := []string{
"docs",
"package.json",
"plugin-testPlugin-v1.2.3-darwin-amd64.zip",
"plugin-testPlugin-v1.2.3-windows-amd64.zip",
}
if diff := cmp.Diff(expect, fileNames(files)); diff != "" {
t.Fatalf("unexpected files in dist directory (-want +got):\n%s", diff)
}

expectPackage := PackageJSON{
SchemaVersion: 1,
Name: "testPlugin",
Type: "destination",
Message: msg,
Version: "v1.2.3",
Protocols: []int{3},
SupportedTargets: []TargetBuild{
{OS: plugin.GoOSWindows, Arch: plugin.GoArchAmd64, Path: "plugin-testPlugin-v1.2.3-windows-amd64.zip", Checksum: "sha256:" + sha256sum(filepath.Join(distDir, "plugin-testPlugin-v1.2.3-windows-amd64.zip"))},
{OS: plugin.GoOSDarwin, Arch: plugin.GoArchAmd64, Path: "plugin-testPlugin-v1.2.3-darwin-amd64.zip", Checksum: "sha256:" + sha256sum(filepath.Join(distDir, "plugin-testPlugin-v1.2.3-darwin-amd64.zip"))},
},
PackageType: plugin.PackageTypeNative,
}
checkPackageJSONContents(t, filepath.Join(distDir, "package.json"), expectPackage)

expectDocs := []string{
"configuration.md",
"overview.md",
}
checkDocs(t, filepath.Join(distDir, "docs"), expectDocs)
})
}
}

func sha256sum(filename string) string {
f, err := os.Open(filename)
if err != nil {
Expand Down

1 comment on commit f6471e3

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⏱️ Benchmark results

  • Glob-8 ns/op: 101.3

Please sign in to comment.