Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| package datatransform | ||
|
|
||
| type DataTransformer interface { | ||
| Modules() []string | ||
| InitializeModule(string) error | ||
| ExecuteModule(string) error | ||
|
|
||
| Close() error | ||
| } | ||
|
|
||
| type NoopDataTransformer struct{} | ||
|
|
||
| var _ DataTransformer = (*NoopDataTransformer)(nil) | ||
|
|
||
| func NewNoopDataTransformer() (*NoopDataTransformer, error) { | ||
| return &NoopDataTransformer{}, nil | ||
| } | ||
|
|
||
| func (n *NoopDataTransformer) Close() error { return nil } | ||
| func (n *NoopDataTransformer) ExecuteModule(string) error { return nil } | ||
| func (n *NoopDataTransformer) InitializeModule(string) error { return nil } | ||
| func (n *NoopDataTransformer) Modules() []string { return []string{} } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,139 @@ | ||
| package datatransform | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "os" | ||
|
|
||
| "github.com/wasmerio/wasmer-go/wasmer" | ||
| "golang.org/x/exp/maps" | ||
| ) | ||
|
|
||
| type Wasmer struct { | ||
| store *wasmer.Store | ||
| modules map[string]*WasiModule | ||
| } | ||
|
|
||
| type WasiModule struct { | ||
| *wasmer.Instance | ||
| *wasmer.WasiEnvironment | ||
| } | ||
|
|
||
| var _ DataTransformer = (*Wasmer)(nil) | ||
|
|
||
| func getRuntimeEngineKind() (wasmer.EngineKind, error) { | ||
| if wasmer.IsEngineAvailable(wasmer.UNIVERSAL) { | ||
| return wasmer.UNIVERSAL, nil | ||
| } | ||
|
|
||
| if wasmer.IsEngineAvailable(wasmer.DYLIB) { | ||
| return wasmer.DYLIB, nil | ||
| } | ||
|
|
||
| return wasmer.EngineKind(0), errors.New("no available wasm engines") | ||
| } | ||
|
|
||
| func getRuntimeCompilerKind() (wasmer.CompilerKind, error) { | ||
| compilersByPriority := []wasmer.CompilerKind{ | ||
| wasmer.CRANELIFT, | ||
| wasmer.LLVM, | ||
| wasmer.SINGLEPASS, | ||
| } | ||
|
|
||
| for _, kind := range compilersByPriority { | ||
| if !wasmer.IsCompilerAvailable(kind) { | ||
| continue | ||
| } | ||
|
|
||
| return kind, nil | ||
| } | ||
|
|
||
| return wasmer.CompilerKind(0), errors.New("no available wasm compilers") | ||
| } | ||
|
|
||
| func NewWasmerDataTransformer() (*Wasmer, error) { | ||
| engineKind, err := getRuntimeEngineKind() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| compilerKind, err := getRuntimeCompilerKind() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| engineConfig := wasmer.NewConfig() | ||
| switch engineKind { | ||
| case wasmer.DYLIB: | ||
| engineConfig.UseDylibEngine() | ||
| default: | ||
| engineConfig.UseJITEngine() | ||
| } | ||
|
|
||
| switch compilerKind { | ||
| case wasmer.CRANELIFT: | ||
| engineConfig.UseCraneliftCompiler() | ||
| case wasmer.LLVM: | ||
| engineConfig.UseLLVMCompiler() | ||
| default: | ||
| engineConfig.UseSinglepassCompiler() | ||
| } | ||
|
|
||
| engine := wasmer.NewEngineWithConfig(engineConfig) | ||
| store := wasmer.NewStore(engine) | ||
| return &Wasmer{store: store, modules: map[string]*WasiModule{}}, nil | ||
| } | ||
|
|
||
| func (w *Wasmer) Close() error { | ||
| w.store.Close() | ||
| return nil | ||
| } | ||
|
|
||
| func (w *Wasmer) Modules() []string { return maps.Keys(w.modules) } | ||
|
|
||
| func (w *Wasmer) InitializeModule(path string) error { | ||
| content, err := os.ReadFile(path) | ||
| if err != nil { | ||
| return fmt.Errorf("unable to load %s: %w", path, err) | ||
| } | ||
|
|
||
| mod, err := wasmer.NewModule(w.store, content) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| env, err := wasmer.NewWasiStateBuilder(path).CaptureStderr().CaptureStdout().Finalize() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| imports, err := env.GenerateImportObject(w.store, mod) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| instance, err := wasmer.NewInstance(mod, imports) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| w.modules[path] = &WasiModule{Instance: instance, WasiEnvironment: env} | ||
| return nil | ||
| } | ||
|
|
||
| func (w *Wasmer) ExecuteModule(path string) error { | ||
| mod, ok := w.modules[path] | ||
| if !ok { | ||
| return fmt.Errorf("requested unknown module %s", path) | ||
| } | ||
|
|
||
| start, err := mod.Exports.GetWasiStartFunction() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| rv, err := start() | ||
| fmt.Println(rv, string(mod.ReadStdout()), string(mod.ReadStderr())) | ||
|
|
||
| // if _, ok := err.(*wasmer.TrapError); ok { | ||
| // return nil | ||
| // } | ||
| return err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| package datatransform | ||
|
|
||
| // type Wazero struct { | ||
| // wazero.Runtime | ||
| // modules map[string]wazero.CompiledModule | ||
| // } | ||
|
|
||
| // var _ DataTransformer = (*Wazero)(nil) | ||
|
|
||
| // func NewWazeroDataTransform(ctx context.Context) (*Wazero, error) { | ||
| // cfg := wazero.NewRuntimeConfig().WithCloseOnContextDone(true) | ||
| // rt := wazero.NewRuntimeWithConfig(ctx, cfg) | ||
|
|
||
| // _, err := wasi_snapshot_preview1.Instantiate(ctx, rt) | ||
| // if err != nil { | ||
| // return nil, err | ||
| // } | ||
|
|
||
| // return &Wazero{Runtime: rt, modules: map[string]wazero.CompiledModule{}}, nil | ||
| // } | ||
|
|
||
| // func (w *Wazero) Close() error { return w.Close() } | ||
| // func (w *Wazero) Modules() []string { return maps.Keys(w.modules) } | ||
|
|
||
| // func (w *Wazero) InitializeModule(ctx context.Context, path string) error { | ||
| // content, err := os.ReadFile(path) | ||
| // if err != nil { | ||
| // return fmt.Errorf("unable to load %s: %w", path, err) | ||
| // } | ||
| // mod, err := w.CompileModule(ctx, content) | ||
| // if err != nil { | ||
| // return err | ||
| // } | ||
| // w.modules[path] = mod | ||
| // return nil | ||
| // } | ||
|
|
||
| // func (w *Wazero) ExecuteModule(path string) error { | ||
| // mod, ok := w.modules[path] | ||
| // if !ok { | ||
| // return fmt.Errorf("requested unknown module %s", path) | ||
| // } | ||
|
|
||
| // fn, ok := mod.ExportedFunctions()["main"] | ||
| // if !ok { | ||
| // return fmt.Errorf("requested unknown function %s in module %s", "main", path) | ||
| // } | ||
|
|
||
| // } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| kind: source | ||
| spec: | ||
| name: "aws" | ||
| path: cloudquery/aws | ||
| registry: cloudquery | ||
| version: v26.10.0 | ||
| destinations: ["postgresql"] | ||
| tables: ["*s3_bucket*"] | ||
| spec: | ||
| accounts: | ||
| - id: "615713231484" | ||
| --- | ||
| kind: destination | ||
| spec: | ||
| name: "postgresql" | ||
| path: cloudquery/postgresql | ||
| registry: cloudquery | ||
| version: v8.0.5 | ||
| write_mode: "overwrite-delete-stale" # overwrite-delete-stale, overwrite, append | ||
| spec: | ||
| connection_string: "postgresql://postgres:12345@localhost:5432/postgres?sslmode=disable" | ||
| --- | ||
| kind: transform | ||
| spec: | ||
| name: test | ||
| path: /Users/alex/Developer/scratch/transform/main.wasm |