From 9463df28c63d8c343cfae52ca041ce88750a474c Mon Sep 17 00:00:00 2001 From: Nick Stogner Date: Sun, 1 Oct 2023 00:52:42 -0400 Subject: [PATCH] Standalone CLI: sub (#232) Add a standalone CLI: `sub` Replaces the current kubectl plugins. --- .github/workflows/integration-tests.yml | 4 + .goreleaser.yaml | 27 +- Dockerfile | 2 +- Dockerfile.sci-gcp | 2 +- Dockerfile.sci-kind | 2 +- Makefile | 2 +- api/v1/conditions.go | 6 +- api/v1/model_types.go | 7 +- api/v1/notebook_types.go | 8 + api/v1/zz_generated.deepcopy.go | 9 +- cmd/controllermanager/main.go | 8 +- cmd/sub/main.go | 16 + config/crd/bases/substratus.ai_models.yaml | 38 +- config/crd/bases/substratus.ai_notebooks.yaml | 6 + containertools/cmd/nbwatch/main.go | 36 +- docs/cli.md | 316 +++++++++++++++ docs/container-contract.md | 10 +- docs/design.md | 26 +- docs/development.md | 18 +- .../finetuned-model-gitops.yaml | 4 +- .../facebook-opt-125m/finetuned-model.yaml | 4 +- examples/falcon-40b/finetuned-model.yaml | 4 +- .../finetuned-model-custom-prompt.yaml | 4 +- .../falcon-7b-instruct/finetuned-model.yaml | 4 +- examples/llama2-7b/finetuned-model.yaml | 4 +- examples/notebook/src/hello.py | 2 +- go.mod | 28 +- go.sum | 73 +++- internal/cli/common.go | 15 + internal/cli/delete.go | 78 ++++ internal/cli/get.go | 85 ++++ internal/cli/infer.go | 82 ++++ internal/cli/notebook.go | 158 ++++++++ internal/cli/root.go | 23 ++ internal/cli/run.go | 89 +++++ internal/cli/serve.go | 80 ++++ .../cli/utils/kubeconfig.go | 24 +- internal/cli/utils/uuid.go | 7 + .../internal => internal}/client/client.go | 33 +- .../client}/cp/kubectl.go | 0 .../client/decode_encode.go | 5 +- internal/client/notebook.go | 86 ++++ .../client/port_forward.go | 24 +- {kubectl/internal => internal}/client/sync.go | 52 ++- .../internal => internal}/client/upload.go | 65 ++- internal/cloud/cloud.go | 3 +- internal/controller/build_reconciler.go | 35 +- internal/controller/dataset_controller.go | 45 ++- .../controller/dataset_controller_test.go | 6 +- internal/controller/main_test.go | 22 +- internal/controller/manager.go | 19 +- internal/controller/model_controller.go | 88 +++-- internal/controller/model_controller_test.go | 11 +- internal/controller/notebook_controller.go | 61 ++- .../controller/notebook_controller_test.go | 5 +- internal/controller/server_controller.go | 15 +- internal/controller/utils.go | 25 +- internal/tui/common.go | 304 +++++++++++++++ internal/tui/delete.go | 162 ++++++++ internal/tui/get.go | 369 ++++++++++++++++++ internal/tui/infer_chat.go | 92 +++++ internal/tui/manifests.go | 187 +++++++++ internal/tui/notebook.go | 330 ++++++++++++++++ internal/tui/pods.go | 246 ++++++++++++ internal/tui/portforward.go | 63 +++ internal/tui/readiness.go | 100 +++++ internal/tui/run.go | 176 +++++++++ internal/tui/serve.go | 281 +++++++++++++ internal/tui/styles.go | 33 ++ internal/tui/upload.go | 169 ++++++++ kubectl/cmd/applybuild/main.go | 15 - kubectl/cmd/notebook/main.go | 15 - kubectl/internal/client/notebook.go | 84 ---- kubectl/internal/commands/applybuild.go | 129 ------ kubectl/internal/commands/applybuild_test.go | 72 ---- kubectl/internal/commands/main_test.go | 146 ------- kubectl/internal/commands/notebook.go | 352 ----------------- kubectl/internal/commands/notebook_test.go | 92 ----- .../commands/test-applybuild/Dockerfile | 6 - .../commands/test-applybuild/hello.txt | 1 - .../commands/test-applybuild/notebook.yaml | 9 - .../commands/test-notebook/Dockerfile | 6 - .../internal/commands/test-notebook/hello.txt | 1 - .../commands/test-notebook/notebook.yaml | 9 - 84 files changed, 4092 insertions(+), 1268 deletions(-) create mode 100644 cmd/sub/main.go create mode 100644 docs/cli.md create mode 100644 internal/cli/common.go create mode 100644 internal/cli/delete.go create mode 100644 internal/cli/get.go create mode 100644 internal/cli/infer.go create mode 100644 internal/cli/notebook.go create mode 100644 internal/cli/root.go create mode 100644 internal/cli/run.go create mode 100644 internal/cli/serve.go rename kubectl/internal/commands/utils.go => internal/cli/utils/kubeconfig.go (64%) create mode 100644 internal/cli/utils/uuid.go rename {kubectl/internal => internal}/client/client.go (69%) rename {kubectl/internal => internal/client}/cp/kubectl.go (100%) rename {kubectl/internal => internal}/client/decode_encode.go (91%) create mode 100644 internal/client/notebook.go rename {kubectl/internal => internal}/client/port_forward.go (53%) rename {kubectl/internal => internal}/client/sync.go (82%) rename {kubectl/internal => internal}/client/upload.go (84%) create mode 100644 internal/tui/common.go create mode 100644 internal/tui/delete.go create mode 100644 internal/tui/get.go create mode 100644 internal/tui/infer_chat.go create mode 100644 internal/tui/manifests.go create mode 100644 internal/tui/notebook.go create mode 100644 internal/tui/pods.go create mode 100644 internal/tui/portforward.go create mode 100644 internal/tui/readiness.go create mode 100644 internal/tui/run.go create mode 100644 internal/tui/serve.go create mode 100644 internal/tui/styles.go create mode 100644 internal/tui/upload.go delete mode 100644 kubectl/cmd/applybuild/main.go delete mode 100644 kubectl/cmd/notebook/main.go delete mode 100644 kubectl/internal/client/notebook.go delete mode 100644 kubectl/internal/commands/applybuild.go delete mode 100644 kubectl/internal/commands/applybuild_test.go delete mode 100644 kubectl/internal/commands/main_test.go delete mode 100644 kubectl/internal/commands/notebook.go delete mode 100644 kubectl/internal/commands/notebook_test.go delete mode 100644 kubectl/internal/commands/test-applybuild/Dockerfile delete mode 100644 kubectl/internal/commands/test-applybuild/hello.txt delete mode 100644 kubectl/internal/commands/test-applybuild/notebook.yaml delete mode 100644 kubectl/internal/commands/test-notebook/Dockerfile delete mode 100644 kubectl/internal/commands/test-notebook/hello.txt delete mode 100644 kubectl/internal/commands/test-notebook/notebook.yaml diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c58a8ac6..ccff10e2 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -8,4 +8,8 @@ jobs: steps: - name: Check out repository code uses: actions/checkout@v3 + - name: Setup Go 1.21.x + uses: actions/setup-go@v4 + with: + go-version: '1.21.x' - run: make test-integration diff --git a/.goreleaser.yaml b/.goreleaser.yaml index c6255626..ec31f9fb 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -6,23 +6,9 @@ before: release: prerelease: "true" builds: - - id: kubectl-applybuild - binary: kubectl-applybuild - main: ./kubectl/cmd/applybuild/ - ldflags: "-X 'main.Version={{.Version}}'" - env: - - CGO_ENABLED=0 - goos: - - linux - - windows - - darwin - goarch: - - amd64 - - arm64 - - arm - - id: kubectl-notebook - binary: kubectl-notebook - main: ./kubectl/cmd/notebook/ + - id: sub + binary: sub + main: ./cmd/sub/ ldflags: "-X 'main.Version={{.Version}}'" env: - CGO_ENABLED=0 @@ -61,13 +47,12 @@ archives: format_overrides: - goos: windows format: zip - - id: kubectl-plugins + - id: sub builds: - - kubectl-applybuild - - kubectl-notebook + - sub format: tar.gz name_template: >- - kubectl-plugins- + sub- {{- .Os }}- {{- .Arch }} # use zip for windows archives diff --git a/Dockerfile b/Dockerfile index 08b61397..c7145776 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.19 as builder +FROM golang:1.21 as builder ARG TARGETOS ARG TARGETARCH diff --git a/Dockerfile.sci-gcp b/Dockerfile.sci-gcp index 5bd6c9c7..28360a2b 100644 --- a/Dockerfile.sci-gcp +++ b/Dockerfile.sci-gcp @@ -1,5 +1,5 @@ # Start from the latest go base image -FROM golang:1.20-bookworm AS builder +FROM golang:1.21-bookworm AS builder ARG TARGETOS=linux ARG TARGETARCH=amd64 diff --git a/Dockerfile.sci-kind b/Dockerfile.sci-kind index b799c15e..c1e4bfba 100644 --- a/Dockerfile.sci-kind +++ b/Dockerfile.sci-kind @@ -1,5 +1,5 @@ # Start from the latest go base image -FROM golang:1.19 AS builder +FROM golang:1.21 AS builder ARG TARGETOS=linux ARG TARGETARCH=amd64 diff --git a/Makefile b/Makefile index cefe485b..924cacfa 100644 --- a/Makefile +++ b/Makefile @@ -189,7 +189,7 @@ dev-down-aws: build-installer .PHONY: dev-run-gcp # Controller manager configuration # dev-run-gcp: export CLOUD=gcp -dev-run-gcp: PROJECT_ID ?= $(shell gcloud config get project) +dev-run-gcp: export PROJECT_ID=$(shell gcloud config get project) dev-run-gcp: export CLUSTER_NAME=substratus dev-run-gcp: export CLUSTER_LOCATION=us-central1 dev-run-gcp: export PRINCIPAL=substratus@${PROJECT_ID}.iam.gserviceaccount.com diff --git a/api/v1/conditions.go b/api/v1/conditions.go index 0def73dc..3b9bdd9a 100644 --- a/api/v1/conditions.go +++ b/api/v1/conditions.go @@ -3,9 +3,8 @@ package v1 const ( ConditionUploaded = "Uploaded" ConditionBuilt = "Built" - ConditionLoaded = "Loaded" - ConditionModelled = "Modelled" - ConditionDeployed = "Deployed" + ConditionComplete = "Complete" + ConditionServing = "Serving" ) const ( @@ -20,6 +19,7 @@ const ( ReasonJobNotComplete = "JobNotComplete" ReasonJobComplete = "JobComplete" + ReasonJobFailed = "JobFailed" ReasonDeploymentReady = "DeploymentReady" ReasonDeploymentNotReady = "DeploymentNotReady" ReasonPodReady = "PodReady" diff --git a/api/v1/model_types.go b/api/v1/model_types.go index 9599dba6..97ffef92 100644 --- a/api/v1/model_types.go +++ b/api/v1/model_types.go @@ -23,12 +23,12 @@ type ModelSpec struct { // Resources are the compute resources required by the container. Resources *Resources `json:"resources,omitempty"` - // BaseModel should be set in order to mount another model to be + // Model should be set in order to mount another model to be // used for transfer learning. - BaseModel *ObjectRef `json:"baseModel,omitempty"` + Model *ObjectRef `json:"model,omitempty"` // Dataset to mount for training. - TrainingDataset *ObjectRef `json:"trainingDataset,omitempty"` + Dataset *ObjectRef `json:"dataset,omitempty"` // Parameters are passing into the model training/loading container as environment variables. // Environment variable name will be `"PARAM_" + uppercase(key)`. @@ -42,6 +42,7 @@ func (m *Model) GetParams() map[string]intstr.IntOrString { func (m *Model) GetBuild() *Build { return m.Spec.Build } + func (m *Model) SetBuild(b *Build) { m.Spec.Build = b } diff --git a/api/v1/notebook_types.go b/api/v1/notebook_types.go index c3604acd..88607412 100644 --- a/api/v1/notebook_types.go +++ b/api/v1/notebook_types.go @@ -44,6 +44,7 @@ func (n *Notebook) GetParams() map[string]intstr.IntOrString { func (n *Notebook) GetBuild() *Build { return n.Spec.Build } + func (n *Notebook) SetBuild(b *Build) { n.Spec.Build = b } @@ -79,6 +80,10 @@ func (n *Notebook) GetStatusUpload() UploadStatus { return n.Status.BuildUpload } +func (n *Notebook) GetStatusArtifacts() ArtifactsStatus { + return n.Status.Artifacts +} + func (n *Notebook) IsSuspended() bool { return n.Spec.Suspend != nil && *n.Spec.Suspend } @@ -92,6 +97,9 @@ type NotebookStatus struct { // Conditions is the list of conditions that describe the current state of the Notebook. Conditions []metav1.Condition `json:"conditions,omitempty"` + // Artifacts status. + Artifacts ArtifactsStatus `json:"artifacts,omitempty"` + // BuildUpload contains the status of the build context upload. BuildUpload UploadStatus `json:"buildUpload,omitempty"` } diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index e0123ce5..d86743f3 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -321,13 +321,13 @@ func (in *ModelSpec) DeepCopyInto(out *ModelSpec) { *out = new(Resources) (*in).DeepCopyInto(*out) } - if in.BaseModel != nil { - in, out := &in.BaseModel, &out.BaseModel + if in.Model != nil { + in, out := &in.Model, &out.Model *out = new(ObjectRef) **out = **in } - if in.TrainingDataset != nil { - in, out := &in.TrainingDataset, &out.TrainingDataset + if in.Dataset != nil { + in, out := &in.Dataset, &out.Dataset *out = new(ObjectRef) **out = **in } @@ -507,6 +507,7 @@ func (in *NotebookStatus) DeepCopyInto(out *NotebookStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + out.Artifacts = in.Artifacts in.BuildUpload.DeepCopyInto(&out.BuildUpload) } diff --git a/cmd/controllermanager/main.go b/cmd/controllermanager/main.go index 3d83e191..4e32db46 100644 --- a/cmd/controllermanager/main.go +++ b/cmd/controllermanager/main.go @@ -7,17 +7,14 @@ import ( "io/ioutil" "os" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "gopkg.in/yaml.v2" - "k8s.io/client-go/kubernetes" - _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -26,7 +23,6 @@ import ( "github.com/substratusai/substratus/internal/cloud" "github.com/substratusai/substratus/internal/controller" "github.com/substratusai/substratus/internal/sci" - //+kubebuilder:scaffold:imports ) var ( diff --git a/cmd/sub/main.go b/cmd/sub/main.go new file mode 100644 index 00000000..cd45ab92 --- /dev/null +++ b/cmd/sub/main.go @@ -0,0 +1,16 @@ +package main + +import ( + "log" + + "github.com/substratusai/substratus/internal/cli" +) + +var Version = "development" + +func main() { + cli.Version = Version + if err := cli.Command().Execute(); err != nil { + log.Fatal(err) + } +} diff --git a/config/crd/bases/substratus.ai_models.yaml b/config/crd/bases/substratus.ai_models.yaml index fe03196d..4fe6f94d 100644 --- a/config/crd/bases/substratus.ai_models.yaml +++ b/config/crd/bases/substratus.ai_models.yaml @@ -44,16 +44,6 @@ spec: spec: description: Spec is the desired state of the Model. properties: - baseModel: - description: BaseModel should be set in order to mount another model - to be used for transfer learning. - properties: - name: - description: Name of Kubernetes object. - type: string - required: - - name - type: object build: description: Build specifies how to build an image. properties: @@ -113,6 +103,15 @@ spec: items: type: string type: array + dataset: + description: Dataset to mount for training. + properties: + name: + description: Name of Kubernetes object. + type: string + required: + - name + type: object env: additionalProperties: type: string @@ -121,6 +120,16 @@ spec: image: description: Image that contains model code and dependencies. type: string + model: + description: Model should be set in order to mount another model to + be used for transfer learning. + properties: + name: + description: Name of Kubernetes object. + type: string + required: + - name + type: object params: additionalProperties: anyOf: @@ -161,15 +170,6 @@ spec: format: int64 type: integer type: object - trainingDataset: - description: Dataset to mount for training. - properties: - name: - description: Name of Kubernetes object. - type: string - required: - - name - type: object type: object status: description: Status is the observed state of the Model. diff --git a/config/crd/bases/substratus.ai_notebooks.yaml b/config/crd/bases/substratus.ai_notebooks.yaml index afa26e35..af679812 100644 --- a/config/crd/bases/substratus.ai_notebooks.yaml +++ b/config/crd/bases/substratus.ai_notebooks.yaml @@ -180,6 +180,12 @@ spec: status: description: Status is the observed state of the Notebook. properties: + artifacts: + description: Artifacts status. + properties: + url: + type: string + type: object buildUpload: description: BuildUpload contains the status of the build context upload. diff --git a/containertools/cmd/nbwatch/main.go b/containertools/cmd/nbwatch/main.go index ed06530e..baa2df02 100644 --- a/containertools/cmd/nbwatch/main.go +++ b/containertools/cmd/nbwatch/main.go @@ -8,14 +8,15 @@ import ( "path/filepath" "strings" - "k8s.io/klog/v2" - "github.com/fsnotify/fsnotify" ) var Version = "development" func main() { + log.SetOutput(os.Stderr) + log.Println("Starting") + if len(os.Args) == 2 && os.Args[1] == "version" { fmt.Printf("nbwatch %s\n", Version) os.Exit(0) @@ -33,7 +34,31 @@ func run() error { } defer w.Close() - w.Add("/content/src") + const contentDir = "/content" + + // NOTE: Watch is non-recursive. + log.Printf("Watching: %v", contentDir) + w.Add(contentDir) + + entries, err := os.ReadDir(contentDir) + if err != nil { + return fmt.Errorf("reading dir: %w", err) + } + for _, e := range entries { + if !e.IsDir() { + continue + } + + switch name := e.Name(); name { + case "data", "model", "artifacts": + default: + if !strings.HasPrefix(name, ".") { + p := filepath.Join(contentDir, name) + log.Printf("Watching: %v", p) + w.Add(p) + } + } + } watchLoop(w) @@ -49,7 +74,7 @@ func watchLoop(w *fsnotify.Watcher) { if !ok { // Channel was closed (i.e. Watcher.Close() was called). return } - klog.Error(err) + log.Printf("error: %v", err) // Read from Events. case e, ok := <-w.Events: if !ok { // Channel was closed (i.e. Watcher.Close() was called). @@ -59,9 +84,10 @@ func watchLoop(w *fsnotify.Watcher) { i++ path := e.Name + base := filepath.Base(path) // Covers ".git", ".gitignore", ".gitmodules", ".gitattributes", ".ipynb_checkpoints" // and also temporary files that Jupyter writes on save like: ".~hello.py" - if strings.HasPrefix(filepath.Base(path), ".") { + if strings.HasPrefix(base, ".") { continue } diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 00000000..202d6dac --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,316 @@ +# CLI + +Operates on directories as a unit of work: + +``` +my-model/ + substratus.yaml <---- Reads Dataset/Model/Server/Notebook from here. + Dockerfile <--- Optional + run.ipynb +``` + +## Alternative Names + +### "ai" CLI + +Pros + +* Short and sweet + +Cons + +* Generic + +```bash +# Looks for "ai.yaml"... + +ai notebook . +ai nb . +ai run . +``` + +### "strat" CLI + +Pros + +* More specific than `sub` + +Cons + +* Longer than `sub` + +```bash +strat notebook . +strat nb . +strat run . +``` + +## Notebook + +```bash +sub notebook . +sub nb . +``` + +## Get + +```bash +sub get +``` + +``` +sub get + +models/ + facebook-opt-125m + falcon-7b + +datasets/ + squad + +servers/ + falcon-7b +``` + +``` +sub get models + +facebook-opt-125m +falcon-7b +``` + +``` +sub get models/falcon-7b + +v3 +v2 +v1 +``` + +``` +sub get models/falcon-7b.v3 + +metrics: + loss: 0.9 + abc: 123 +``` + +## Apply + +``` +# Alternative names??? + +# "run" --> currently prefer "apply" b/c the target is a noun/end-object (Model / Dataset / Server) +sub run . +``` + +### Apply (with `` arg) + +* Tar & upload +* Remote build +* Wait for Ready + +```bash +sub apply . +``` + +## View + +* Grab `run.html` (converted notebook) and serve on localhost. +* Open browser. + +```bash +sub view model/falcon-7b +sub view dataset/squad +``` + +Alternative names: + +```bash +sub logs +sub show +sub inspect +``` + +## Delete + +```bash +sub delete / +sub del + +# By name +sub delete models/facebook-opt-125m +sub delete datasets/squad +``` + +## Inference Client + +```bash +sub infer +sub inf + +# OR: +sub cl (client) +sub ch (chat) # LLM prompt/completion +sub cl (classify) # Image recognition +``` + +https://github.com/charmbracelet/bubbletea/tree/master/examples#chat + + +## Substratus.yaml + +### Option 1: Workspace file + +A "workspace" file could represent multiple different objects. + +```yaml +apiVersion: substratus.ai/v1 +metadata: + name: snakes-opt-125m +model: + dataset: + name: snakes + model: + name: facebook-opt-125m +dataset: +server: + # Generated if not specified + # Only valid if model is specified. +notebook: + # Generated if not specified +``` + +### Option 2: Multi-doc + +**Pros:** + +* No new objects + +**Cons:** + +* Duplication of fields like `.metadata` + +```yaml +apiVersion: substratus.ai/v1 +kind: Model +metadata: + name: snakes-opt-125m +spec: + dataset: + name: snakes + model: + name: facebook-opt-125m +--- +apiVersion: substratus.ai/v1 +kind: Server +metadata: + name: snakes-opt-125m +spec: + # ... +--- +apiVersion: substratus.ai/v1 +kind: Notebook +metadata: + name: snakes-opt-125m +spec: + # ... +``` + +### Option 3: Directory + +**Pros:** + +* No new objects +* Options for more than 1 objects + +**Cons:** + +* Duplication of fields like `.metadata` +* Messy + +``` +.substratus/ + dataset.yaml + model.yaml + notebook.yaml + server.yaml + +my-code.ipynb +``` + +```yaml +apiVersion: substratus.ai/v1 +kind: Model +metadata: + name: snakes-opt-125m +spec: + dataset: + name: snakes + model: + name: facebook-opt-125m +--- +apiVersion: substratus.ai/v1 +kind: Server +metadata: + name: snakes-opt-125m +spec: + # ... +--- +apiVersion: substratus.ai/v1 +kind: Notebook +metadata: + name: snakes-opt-125m +spec: + # ... +``` + +### Option 4: Kustomize-like + +**Pros:** + +* No new objects +* Options for more than 1 objects +* Ability to express remote base-model/dataset dependencies + +**Cons:** + +* Could get messy + +``` +substratus.yaml + +dataset.yaml +model.yaml +notebook.yaml +server.yaml + +my-code.ipynb +``` + +```yaml +Model: model.yaml +dependencies: +- file: ./model.yaml +- https: //raw.githubusercontent.com/substratusai/substratus/main/install/kind/manifests.yaml +- gcs: /some-bucket/some/file.yaml +``` + +### Option 5: Multi-doc with remotes + +```yaml +apiVersion: substratus.ai/v1 +kind: Model +metadata: + name: snakes-opt-125m +spec: + dataset: + name: snakes + model: + name: facebook-opt-125m +--- +- ../base-model.yaml +- https://raw.githubusercontent.com/substratusai/substratus/main/examples/facebook-opt-125m.yaml +- https://raw.githubusercontent.com/substratusai/substratus/main/examples/squad-dataset.yaml +``` + diff --git a/docs/container-contract.md b/docs/container-contract.md index d643f7a4..a947ffff 100644 --- a/docs/container-contract.md +++ b/docs/container-contract.md @@ -25,12 +25,10 @@ Note: This requirement is satisfied by default when using Substratus base images ## Directory Structure ``` -/content/ # Working directory. - data/ # Location where Datasets will be mounted (reading and loading). - src/ # Source code (*.py, *.ipynb) for loading, training, etc. - logs/ # Output of building/training jobs for debugging purposes. - model/ # Location to store the resulting model from loading or training. - saved-model/ # Location where a previously saved model will be mounted. +/content/ # Working directory. + data/ # Location where a previously stored Datasets is mounted. + model/ # Location where a previously stored Model is mounted. + artifacts/ # Location to store output of a run. ``` ## Parameters diff --git a/docs/design.md b/docs/design.md index 0dcd0567..8e4b68c4 100644 --- a/docs/design.md +++ b/docs/design.md @@ -113,11 +113,9 @@ The following scheme can be used for storing artifacts for Models, Datasets, and ```sh # Models gs://{bucket}/{hash}/model # Model artifacts (*.pt, etc) -gs://{bucket}/{hash}/logs # Logs and converted notebooks # Datasets gs://{bucket}/{hash}/data # Data artifacts (*.jsonl, etc) -gs://{bucket}/{hash}/logs # Logs and converted notebooks # Notebooks gs://{bucket}/{hash}/build/{md5-checksum}.tar # Uploaded build context @@ -127,7 +125,6 @@ The example Model's backing storage would end up being: ```sh gs://abc123-substratus-artifacts/f94a0d128bcbd9c1b824e9e5572baf86/model/ -gs://abc123-substratus-artifacts/f94a0d128bcbd9c1b824e9e5572baf86/logs/ ``` The Model would report this URL in its status: @@ -166,15 +163,12 @@ runJob() All mount points will are made under a standardized `/content` directory which should correspond to the `WORKDIR` of a Dockerfile. This works well for Jupyter notebooks which can be run with `/content` set as the serving directory: all relevant mounts will be populated on the file explorer sidebar. -The `logs/` directories below are used to store the notebook cell output, python logs, tensorboard stats, etc. These directories are mounted as read-only in Notebooks to explore the output of other background jobs. - ##### Dataset (importing) ``` /content/params.json # Populated from .spec.params (also available as env vars). -/content/data/ # Mounted RW: initially empty dir to write new files -/content/logs/ # Mounted RW: initially empty dir to write new files +/content/artifacts/ # Mounted RW: initially empty dir to write new files ``` ##### Model (importing) @@ -182,8 +176,7 @@ The `logs/` directories below are used to store the notebook cell output, python ``` /content/params.json # Populated from .spec.params (also available as env vars). -/content/model/ # Mounted RW: initially empty dir to write new files -/content/logs/ # Mounted RW: initially empty dir to write new files +/content/artifacts/ # Mounted RW: initially empty dir to write new files ``` ##### Model (training) @@ -191,12 +184,11 @@ The `logs/` directories below are used to store the notebook cell output, python ``` /content/params.json # Populated from .spec.params (also available as env vars). -/content/data/ # Mounted RO: from .spec.trainingDataset +/content/data/ # Mounted RO: from .spec.dataset -/content/saved-model/ # Mounted RO: from .spec.baseModel +/content/model/ # Mounted RO: from .spec.model -/content/model/ # Mounted RW: initially empty dir for writing new files -/content/logs/ # Mounted RW: initially empty dir for writing logs +/content/artifacts/ # Mounted RW: initially empty dir for writing new files ``` ##### Notebook @@ -207,10 +199,10 @@ NOTE: The `saved-model/` directory is the same as the container for the Model ob /content/params.json # Populated from .spec.params (also available as env vars). /content/data/ # Mounted RO: from .spec.dataset -/content/data-logs/ # Mounted RO: from .spec.dataset -/content/saved-model/ # Mounted RO: from .spec.model -/content/saved-model-logs/ # Mounted RO: from .spec.model +/content/model/ # Mounted RO: from .spec.model + +/content/artifacts/ # Mounted RW: initially empty dir for writing new files ``` ##### Server: @@ -218,7 +210,7 @@ NOTE: The `saved-model/` directory is the same as the container for the Model ob ``` /content/params.json # Populated from .spec.params (also available as env vars). -/content/saved-model/ # Mounted RO: from .spec.model +/content/model/ # Mounted RO: from .spec.model ``` ### Naming diff --git a/docs/development.md b/docs/development.md index 41ae51bc..c5253680 100644 --- a/docs/development.md +++ b/docs/development.md @@ -38,19 +38,35 @@ go build ./kubectl/cmd/applybuild && The `kubectl notebook` command depends on container-tools for live-syncing. The plugin will try to download these tools from GitHub releases if they dont already exist with the right versions. -You can build the container-tools for development purposes using the following. NOTE: This is the default cache directory on a mac, this will be different on other machine types. +You can build the container-tools for development purposes using the following. +Mac steps: ```sh export NODE_ARCH=amd64 rm -rf /Users/$USER/Library/Caches/substratus mkdir -p /Users/$USER/Library/Caches/substratus/container-tools/$NODE_ARCH GOOS=linux GOARCH=$NODE_ARCH go build ./containertools/cmd/nbwatch + mv nbwatch /Users/$USER/Library/Caches/substratus/container-tools/$NODE_ARCH/ echo "development" > /Users/$USER/Library/Caches/substratus/container-tools/version.txt ``` +Linux steps: +```sh +export NODE_ARCH=amd64 + +rm -rf ~/.cache/substratus +mkdir -p ~/.cache/substratus/container-tools/$NODE_ARCH +GOOS=linux GOARCH=$NODE_ARCH go build ./containertools/cmd/nbwatch + +mv nbwatch ~/.cache/substratus/container-tools/amd64/ + +echo "development" > ~/.cache/substratus/container-tools/version.txt +``` + + ### Install from release Release binaries are created for most architectures when the repo is tagged. diff --git a/examples/facebook-opt-125m/finetuned-model-gitops.yaml b/examples/facebook-opt-125m/finetuned-model-gitops.yaml index ec030699..50b79e17 100644 --- a/examples/facebook-opt-125m/finetuned-model-gitops.yaml +++ b/examples/facebook-opt-125m/finetuned-model-gitops.yaml @@ -7,9 +7,9 @@ spec: git: url: https://github.com/substratusai/images path: model-trainer-huggingface - baseModel: + model: name: facebook-opt-125m - trainingDataset: + dataset: name: squad params: epochs: 1 diff --git a/examples/facebook-opt-125m/finetuned-model.yaml b/examples/facebook-opt-125m/finetuned-model.yaml index 78c89262..8b102005 100644 --- a/examples/facebook-opt-125m/finetuned-model.yaml +++ b/examples/facebook-opt-125m/finetuned-model.yaml @@ -4,9 +4,9 @@ metadata: name: fb-opt-125m-squad spec: image: substratusai/model-trainer-huggingface - baseModel: + model: name: facebook-opt-125m - trainingDataset: + dataset: name: squad params: epochs: 1 diff --git a/examples/falcon-40b/finetuned-model.yaml b/examples/falcon-40b/finetuned-model.yaml index 2ae3fc3b..8928468d 100644 --- a/examples/falcon-40b/finetuned-model.yaml +++ b/examples/falcon-40b/finetuned-model.yaml @@ -4,9 +4,9 @@ metadata: name: falcon-40b-k8s spec: image: substratusai/model-trainer-huggingface - baseModel: + model: name: falcon-40b - trainingDataset: + dataset: name: k8s-instructions params: epochs: 1 diff --git a/examples/falcon-7b-instruct/finetuned-model-custom-prompt.yaml b/examples/falcon-7b-instruct/finetuned-model-custom-prompt.yaml index 21e77f7d..fd78fc97 100644 --- a/examples/falcon-7b-instruct/finetuned-model-custom-prompt.yaml +++ b/examples/falcon-7b-instruct/finetuned-model-custom-prompt.yaml @@ -4,9 +4,9 @@ metadata: name: falcon-7b-instruct-k8s-custom-prompt spec: image: substratusai/model-trainer-huggingface - baseModel: + model: name: falcon-7b-instruct - trainingDataset: + dataset: name: k8s-instructions params: num_train_epochs: 1 diff --git a/examples/falcon-7b-instruct/finetuned-model.yaml b/examples/falcon-7b-instruct/finetuned-model.yaml index a63113ec..60220136 100644 --- a/examples/falcon-7b-instruct/finetuned-model.yaml +++ b/examples/falcon-7b-instruct/finetuned-model.yaml @@ -4,9 +4,9 @@ metadata: name: falcon-7b-instruct-k8s spec: image: substratusai/model-trainer-huggingface - baseModel: + model: name: falcon-7b-instruct - trainingDataset: + dataset: name: k8s-instructions params: # See HuggingFace transformers.TrainingArguments for all parameters diff --git a/examples/llama2-7b/finetuned-model.yaml b/examples/llama2-7b/finetuned-model.yaml index 7327f428..a4339cb4 100644 --- a/examples/llama2-7b/finetuned-model.yaml +++ b/examples/llama2-7b/finetuned-model.yaml @@ -4,9 +4,9 @@ metadata: name: llama-2-7b-k8s spec: image: substratusai/model-trainer-huggingface - baseModel: + model: name: llama-2-7b - trainingDataset: + dataset: name: k8s-instructions params: # See HuggingFace transformers.TrainingArguments for all parameters diff --git a/examples/notebook/src/hello.py b/examples/notebook/src/hello.py index 8c35c8ee..4039db7a 100644 --- a/examples/notebook/src/hello.py +++ b/examples/notebook/src/hello.py @@ -1 +1 @@ -print("hello!!!") +print("hello!!!") \ No newline at end of file diff --git a/go.mod b/go.mod index 84656a75..8b64ba65 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,21 @@ module github.com/substratusai/substratus -go 1.19 +go 1.21 require ( cloud.google.com/go/compute/metadata v0.2.3 cloud.google.com/go/storage v1.31.0 - github.com/briandowns/spinner v1.23.0 + github.com/aws/aws-sdk-go v1.44.321 + github.com/charmbracelet/bubbles v0.16.1 + github.com/charmbracelet/bubbletea v0.24.2 + github.com/charmbracelet/lipgloss v0.8.0 github.com/go-logr/logr v1.2.4 github.com/go-playground/validator/v10 v10.14.1 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 github.com/sethvargo/go-envconfig v0.9.0 github.com/spf13/cobra v1.6.0 github.com/stretchr/testify v1.8.4 + golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb google.golang.org/grpc v1.57.0 google.golang.org/protobuf v1.31.0 k8s.io/api v0.27.4 @@ -25,9 +29,12 @@ require ( require github.com/inconshreveable/mousetrap v1.0.1 // indirect require ( - github.com/aws/aws-sdk-go v1.44.321 // indirect cloud.google.com/go v0.110.6 // indirect cloud.google.com/go/compute v1.23.0 // indirect + github.com/atotto/clipboard v0.1.4 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/charmbracelet/harmonica v0.2.0 // indirect + github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/go-errors/errors v1.4.2 // indirect @@ -35,15 +42,22 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.15.2 // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xlab/treeprint v1.1.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect golang.org/x/crypto v0.12.0 // indirect - golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect @@ -59,7 +73,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/fatih/color v1.7.0 // indirect github.com/fsnotify/fsnotify v1.6.0 github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect @@ -77,8 +90,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-colorable v0.1.2 // indirect - github.com/mattn/go-isatty v0.0.8 // indirect + github.com/mattn/go-isatty v0.0.18 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/go.sum b/go.sum index 36ad428e..a1da0bf8 100644 --- a/go.sum +++ b/go.sum @@ -13,18 +13,29 @@ cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= +github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aws/aws-sdk-go v1.44.321 h1:iXwFLxWjZPjYqjPq0EcCs46xX7oDLEELte1+BzgpKk8= github.com/aws/aws-sdk-go v1.44.321/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A= -github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY= +github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc= +github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY= +github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg= +github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= +github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= +github.com/charmbracelet/lipgloss v0.8.0 h1:IS00fk4XAHcf8uZKc3eHeMUTCxUH6NkaTrdyCQk84RU= +github.com/charmbracelet/lipgloss v0.8.0/go.mod h1:p4eYUZZJ/0oXTuCQKFF8mqyKCz0ja6y+7DniDDw5KKU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -35,6 +46,8 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= +github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -53,8 +66,6 @@ github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= @@ -62,10 +73,6 @@ github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -78,6 +85,7 @@ github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -85,6 +93,7 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k= github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -123,7 +132,9 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= @@ -144,34 +155,36 @@ github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= +github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= @@ -183,11 +196,20 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b h1:1XF24mVaiu7u+CFywTdcDo2ie1pzzhwjt6RHqzpMU34= +github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= +github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= +github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -204,10 +226,15 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-envconfig v0.9.0 h1:Q6FQ6hVEeTECULvkJZakq3dZMeBQ3JUpcKMfPQbKMDE= github.com/sethvargo/go-envconfig v0.9.0/go.mod h1:Iz1Gy1Sf3T64TQlJSvee81qDhf7YIlt8GMUX6yyNFs0= github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= @@ -243,6 +270,7 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -265,8 +293,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -283,6 +309,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -301,7 +328,6 @@ golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -315,10 +341,13 @@ golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -327,6 +356,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= @@ -342,6 +372,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/cli/common.go b/internal/cli/common.go new file mode 100644 index 00000000..9bb8628d --- /dev/null +++ b/internal/cli/common.go @@ -0,0 +1,15 @@ +package cli + +import ( + "k8s.io/client-go/kubernetes/scheme" + + apiv1 "github.com/substratusai/substratus/api/v1" + "github.com/substratusai/substratus/internal/client" +) + +func init() { + apiv1.AddToScheme(scheme.Scheme) +} + +// NewClient is a dirty hack to allow the client to be mocked out in tests. +var NewClient = client.NewClient diff --git a/internal/cli/delete.go b/internal/cli/delete.go new file mode 100644 index 00000000..b576ed6c --- /dev/null +++ b/internal/cli/delete.go @@ -0,0 +1,78 @@ +package cli + +import ( + "fmt" + "os" + + tea "github.com/charmbracelet/bubbletea" + "github.com/spf13/cobra" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/substratusai/substratus/internal/cli/utils" + "github.com/substratusai/substratus/internal/tui" +) + +func deleteCommand() *cobra.Command { + var flags struct { + namespace string + filename string + kubeconfig string + } + + run := func(cmd *cobra.Command, args []string) error { + defer tui.LogFile.Close() + + kubeconfigNamespace, restConfig, err := utils.BuildConfigFromFlags("", flags.kubeconfig) + if err != nil { + return fmt.Errorf("rest config: %w", err) + } + + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("clientset: %w", err) + } + + c := NewClient(clientset, restConfig) + + // Initialize our program + tui.P = tea.NewProgram((&tui.DeleteModel{ + Ctx: cmd.Context(), + Scope: args[0], + Namespace: tui.Namespace{ + Contextual: kubeconfigNamespace, + Specified: flags.namespace, + }, + Client: c, + }).New()) + if _, err := tui.P.Run(); err != nil { + return err + } + + return nil + } + + cmd := &cobra.Command{ + Use: "delete", + Aliases: []string{"del"}, + Short: "Delete a Substratus Dataset, Model, Server, or Notebook", + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + if err := run(cmd, args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + }, + } + + defaultKubeconfig := os.Getenv("KUBECONFIG") + if defaultKubeconfig == "" { + defaultKubeconfig = clientcmd.RecommendedHomeFile + } + cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "") + + cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook") + cmd.Flags().StringVarP(&flags.filename, "filename", "f", "", "Manifest file") + + return cmd +} diff --git a/internal/cli/get.go b/internal/cli/get.go new file mode 100644 index 00000000..0db3387c --- /dev/null +++ b/internal/cli/get.go @@ -0,0 +1,85 @@ +package cli + +import ( + "fmt" + "os" + + tea "github.com/charmbracelet/bubbletea" + "github.com/spf13/cobra" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/substratusai/substratus/internal/cli/utils" + "github.com/substratusai/substratus/internal/tui" +) + +func getCommand() *cobra.Command { + var flags struct { + namespace string + kubeconfig string + } + + run := func(cmd *cobra.Command, args []string) error { + defer tui.LogFile.Close() + + kubeconfigNamespace, restConfig, err := utils.BuildConfigFromFlags("", flags.kubeconfig) + if err != nil { + return fmt.Errorf("rest config: %w", err) + } + + namespace := "default" + if flags.namespace != "" { + namespace = flags.namespace + } else if kubeconfigNamespace != "" { + namespace = kubeconfigNamespace + } + + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("clientset: %w", err) + } + + c := NewClient(clientset, restConfig) + + var scope string + if len(args) > 0 { + scope = args[0] + } + + // Initialize our program + tui.P = tea.NewProgram((&tui.GetModel{ + Ctx: cmd.Context(), + Scope: scope, + Namespace: namespace, + + Client: c, + }).New() /*, tea.WithAltScreen()*/) + if _, err := tui.P.Run(); err != nil { + return err + } + + return nil + } + + cmd := &cobra.Command{ + Use: "get", + Short: "Get Substratus Datasets, Models, Notebooks, and Servers", + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + if err := run(cmd, args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + }, + } + + defaultKubeconfig := os.Getenv("KUBECONFIG") + if defaultKubeconfig == "" { + defaultKubeconfig = clientcmd.RecommendedHomeFile + } + cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "") + + cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook") + + return cmd +} diff --git a/internal/cli/infer.go b/internal/cli/infer.go new file mode 100644 index 00000000..b6b30df6 --- /dev/null +++ b/internal/cli/infer.go @@ -0,0 +1,82 @@ +package cli + +import ( + "fmt" + "os" + + tea "github.com/charmbracelet/bubbletea" + "github.com/spf13/cobra" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/substratusai/substratus/internal/cli/utils" + "github.com/substratusai/substratus/internal/tui" +) + +func inferCommand() *cobra.Command { + var flags struct { + namespace string + kubeconfig string + } + + run := func(cmd *cobra.Command, args []string) error { + defer tui.LogFile.Close() + + kubeconfigNamespace, restConfig, err := utils.BuildConfigFromFlags("", flags.kubeconfig) + if err != nil { + return fmt.Errorf("rest config: %w", err) + } + + namespace := "default" + if flags.namespace != "" { + namespace = flags.namespace + } else if kubeconfigNamespace != "" { + namespace = kubeconfigNamespace + } + + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("clientset: %w", err) + } + + _ = namespace + + c := NewClient(clientset, restConfig) + _ = c + + // Initialize our program + // TODO: Use a differnt tui-model for different types of Model objects: + // ex: Vector-search TUI, Image recongnition TUI + m := tui.ChatModel{} + + tui.P = tea.NewProgram(m) + if _, err := tui.P.Run(); err != nil { + return err + } + + return nil + } + + cmd := &cobra.Command{ + Use: "infer", + Aliases: []string{"in"}, + Short: "Run inference against a Substratus Server", + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + if err := run(cmd, args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + }, + } + + defaultKubeconfig := os.Getenv("KUBECONFIG") + if defaultKubeconfig == "" { + defaultKubeconfig = clientcmd.RecommendedHomeFile + } + cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "") + + cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook") + + return cmd +} diff --git a/internal/cli/notebook.go b/internal/cli/notebook.go new file mode 100644 index 00000000..a8abf554 --- /dev/null +++ b/internal/cli/notebook.go @@ -0,0 +1,158 @@ +package cli + +import ( + "fmt" + "os" + + tea "github.com/charmbracelet/bubbletea" + "github.com/spf13/cobra" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/substratusai/substratus/internal/cli/utils" + "github.com/substratusai/substratus/internal/tui" +) + +func notebookCommand() *cobra.Command { + var flags struct { + resume string + namespace string + filename string + kubeconfig string + fullscreen bool + } + + run := func(cmd *cobra.Command, args []string) error { + defer tui.LogFile.Close() + + //if flags.filename == "" { + // defaultFilename := "notebook.yaml" + // if _, err := os.Stat(filepath.Join(args[0], "notebook.yaml")); err == nil { + // flags.filename = defaultFilename + // } else { + // return fmt.Errorf("Flag -f (--filename) required when default notebook.yaml file does not exist") + // } + //} + + kubeconfigNamespace, restConfig, err := utils.BuildConfigFromFlags("", flags.kubeconfig) + if err != nil { + return fmt.Errorf("rest config: %w", err) + } + + //namespace := "default" + //if flags.namespace != "" { + // namespace = flags.namespace + //} else if kubeconfigNamespace != "" { + // namespace = kubeconfigNamespace + //} + + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("clientset: %w", err) + } + + c := NewClient(clientset, restConfig) + //notebooks, err := c.Resource(&apiv1.Notebook{ + // TypeMeta: metav1.TypeMeta{ + // APIVersion: "substratus.ai/v1", + // Kind: "Notebook", + // }, + //}) + //if err != nil { + // return fmt.Errorf("resource client: %w", err) + //} + + //var obj client.Object + //if flags.resume != "" { + // fetched, err := notebooks.Get(namespace, flags.resume) + // if err != nil { + // return fmt.Errorf("getting notebook: %w", err) + // } + // obj = fetched.(client.Object) + //} else { + // manifest, err := os.ReadFile(flags.filename) + // if err != nil { + // return fmt.Errorf("reading file: %w", err) + // } + // obj, err = client.Decode(manifest) + // if err != nil { + // return fmt.Errorf("decoding: %w", err) + // } + // if obj.GetNamespace() == "" { + // // When there is no .metadata.namespace set in the manifest... + // obj.SetNamespace(namespace) + // } else { + // // TODO: Closer match kubectl behavior here by differentiaing between + // // the short -n and long --namespace flags. + // // See example kubectl error: + // // error: the namespace from the provided object "a" does not match the namespace "b". You must pass '--namespace=a' to perform this operation. + // if flags.namespace != "" && flags.namespace != obj.GetNamespace() { + // // When there is .metadata.namespace set in the manifest and + // // a conflicting -n or --namespace flag... + // return fmt.Errorf("the namespace from the provided object %q does not match the namespace %q from flag", obj.GetNamespace(), flags.namespace) + // } + // } + //} + + //nb, err := client.NotebookForObject(obj) + //if err != nil { + // return fmt.Errorf("notebook for object: %w", err) + //} + //nb.Spec.Suspend = ptr.To(false) + + var pOpts []tea.ProgramOption + if flags.fullscreen { + pOpts = append(pOpts, tea.WithAltScreen()) + } + + path := "." + if len(args) > 0 { + path = args[0] + } + + // Initialize our program + tui.P = tea.NewProgram((&tui.NotebookModel{ + Ctx: cmd.Context(), + Path: path, + Filename: flags.filename, + Namespace: tui.Namespace{ + Contextual: kubeconfigNamespace, + Specified: flags.namespace, + }, + Client: c, + K8s: clientset, + }).New(), pOpts...) + if _, err := tui.P.Run(); err != nil { + return err + } + + return nil + } + + cmd := &cobra.Command{ + Use: "notebook [dir]", + Aliases: []string{"nb"}, + Short: "Start a Jupyter Notebook development environment", + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + if err := run(cmd, args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + }, + } + + defaultKubeconfig := os.Getenv("KUBECONFIG") + if defaultKubeconfig == "" { + defaultKubeconfig = clientcmd.RecommendedHomeFile + } + cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "") + + cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook") + cmd.Flags().StringVarP(&flags.filename, "filename", "f", "", "Manifest file") + cmd.Flags().StringVarP(&flags.resume, "resume", "r", "", "Name of notebook to resume") + + cmd.Flags().BoolVar(&flags.fullscreen, "fullscreen", false, "Fullscreen mode") + + return cmd +} diff --git a/internal/cli/root.go b/internal/cli/root.go new file mode 100644 index 00000000..c4ad1ab5 --- /dev/null +++ b/internal/cli/root.go @@ -0,0 +1,23 @@ +package cli + +import ( + "github.com/spf13/cobra" +) + +var Version string + +func Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "sub", + Short: "Substratus CLI", + } + + cmd.AddCommand(notebookCommand()) + cmd.AddCommand(runCommand()) + cmd.AddCommand(getCommand()) + // cmd.AddCommand(inferCommand()) + cmd.AddCommand(deleteCommand()) + cmd.AddCommand(serveCommand()) + + return cmd +} diff --git a/internal/cli/run.go b/internal/cli/run.go new file mode 100644 index 00000000..dbcb0861 --- /dev/null +++ b/internal/cli/run.go @@ -0,0 +1,89 @@ +package cli + +import ( + "fmt" + "os" + + tea "github.com/charmbracelet/bubbletea" + "github.com/spf13/cobra" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/substratusai/substratus/internal/cli/utils" + "github.com/substratusai/substratus/internal/tui" +) + +func runCommand() *cobra.Command { + var flags struct { + namespace string + filename string + kubeconfig string + } + + run := func(cmd *cobra.Command, args []string) error { + defer tui.LogFile.Close() + + kubeconfigNamespace, restConfig, err := utils.BuildConfigFromFlags("", flags.kubeconfig) + if err != nil { + return fmt.Errorf("rest config: %w", err) + } + + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("clientset: %w", err) + } + + path := "." + if len(args) > 0 { + path = args[0] + } + + tui.P = tea.NewProgram((&tui.RunModel{ + Ctx: cmd.Context(), + Path: path, + Filename: flags.filename, + Namespace: tui.Namespace{ + Contextual: kubeconfigNamespace, + Specified: flags.namespace, + }, + Client: NewClient(clientset, restConfig), + K8s: clientset, + }).New()) + if _, err := tui.P.Run(); err != nil { + return err + } + + return nil + } + + cmd := &cobra.Command{ + Use: "run [dir]", + Short: "Run a local directory. Supported kinds: Dataset, Model.", + Example: ` # Upload code from the current directory, + # scan *.yaml files looking for Substratus manifests to use. + sub run + + # Upload modelling code and create a Model. + sub run -f model.yaml . + + # Upoad dataset importing code and create a Dataset. + sub run -f dataset.yaml .`, + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + if err := run(cmd, args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + }, + } + + defaultKubeconfig := os.Getenv("KUBECONFIG") + if defaultKubeconfig == "" { + defaultKubeconfig = clientcmd.RecommendedHomeFile + } + cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "path to Kubernetes Kubeconfig file") + cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "namespace of Notebook") + cmd.Flags().StringVarP(&flags.filename, "filename", "f", "", "manifest file") + + return cmd +} diff --git a/internal/cli/serve.go b/internal/cli/serve.go new file mode 100644 index 00000000..8e42b850 --- /dev/null +++ b/internal/cli/serve.go @@ -0,0 +1,80 @@ +package cli + +import ( + "fmt" + "os" + + tea "github.com/charmbracelet/bubbletea" + "github.com/spf13/cobra" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/substratusai/substratus/internal/cli/utils" + "github.com/substratusai/substratus/internal/tui" +) + +func serveCommand() *cobra.Command { + var flags struct { + namespace string + filename string + kubeconfig string + } + + run := func(cmd *cobra.Command, args []string) error { + defer tui.LogFile.Close() + + //if flags.filename == "" { + // return fmt.Errorf("Flag -f (--filename) required") + //} + + kubeconfigNamespace, restConfig, err := utils.BuildConfigFromFlags("", flags.kubeconfig) + if err != nil { + return fmt.Errorf("rest config: %w", err) + } + + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("clientset: %w", err) + } + + // Initialize our program + tui.P = tea.NewProgram((&tui.ServeModel{ + Ctx: cmd.Context(), + Filename: flags.filename, + Namespace: tui.Namespace{ + Contextual: kubeconfigNamespace, + Specified: flags.namespace, + }, + Client: NewClient(clientset, restConfig), + K8s: clientset, + }).New()) + if _, err := tui.P.Run(); err != nil { + return err + } + + return nil + } + + cmd := &cobra.Command{ + Use: "serve", + Aliases: []string{"srv"}, + Short: "Serve a model, open a browser", + Run: func(cmd *cobra.Command, args []string) { + if err := run(cmd, args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + }, + } + + defaultKubeconfig := os.Getenv("KUBECONFIG") + if defaultKubeconfig == "" { + defaultKubeconfig = clientcmd.RecommendedHomeFile + } + + cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "") + cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook") + cmd.Flags().StringVarP(&flags.filename, "filename", "f", "", "Manifest file") + + return cmd +} diff --git a/kubectl/internal/commands/utils.go b/internal/cli/utils/kubeconfig.go similarity index 64% rename from kubectl/internal/commands/utils.go rename to internal/cli/utils/kubeconfig.go index 2e993828..9c7f26e9 100644 --- a/kubectl/internal/commands/utils.go +++ b/internal/cli/utils/kubeconfig.go @@ -1,34 +1,16 @@ -package commands +package utils import ( - "io" - "os" - - "github.com/google/uuid" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/klog/v2" - - "github.com/substratusai/substratus/kubectl/internal/client" ) -var Version = "development" - -// NewClient is a dirty hack to allow the client to be mocked out in tests. -var NewClient = client.NewClient - -// NotebookStdout is a dirty hack to allow stdout to be inspected in tests. -var NotebookStdout io.Writer = os.Stdout - -var NewUUID = func() string { - return uuid.New().String() -} - -// buildConfigFromFlags is a modified version of clientcmd.BuildConfigFromFlags +// BuildConfigFromFlags is a modified version of clientcmd.BuildConfigFromFlags // that returns the namespace set in the kubeconfig to make sure we play nicely // with tools like kubens. -func buildConfigFromFlags(masterUrl, kubeconfigPath string) (string, *restclient.Config, error) { +func BuildConfigFromFlags(masterUrl, kubeconfigPath string) (string, *restclient.Config, error) { if kubeconfigPath == "" && masterUrl == "" { klog.Warning("Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work.") kubeconfig, err := restclient.InClusterConfig() diff --git a/internal/cli/utils/uuid.go b/internal/cli/utils/uuid.go new file mode 100644 index 00000000..11f6959b --- /dev/null +++ b/internal/cli/utils/uuid.go @@ -0,0 +1,7 @@ +package utils + +import "github.com/google/uuid" + +var NewUUID = func() string { + return uuid.New().String() +} diff --git a/kubectl/internal/client/client.go b/internal/client/client.go similarity index 69% rename from kubectl/internal/client/client.go rename to internal/client/client.go index a4c81d8e..8311805a 100644 --- a/kubectl/internal/client/client.go +++ b/internal/client/client.go @@ -3,11 +3,16 @@ package client import ( "context" "fmt" + "io" "time" meta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" "k8s.io/cli-runtime/pkg/resource" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -31,9 +36,12 @@ func init() { var _ Interface = &Client{} type Interface interface { - PortForwardNotebook(ctx context.Context, verbose bool, nb *apiv1.Notebook, ready chan struct{}) error + PortForward(ctx context.Context, logger io.Writer, podRef types.NamespacedName, ports ForwardedPorts, ready chan struct{}) error Resource(obj Object) (*Resource, error) - SyncFilesFromNotebook(context.Context, *apiv1.Notebook, string) error + SyncFilesFromNotebook(context.Context, *apiv1.Notebook, string, + io.Writer, + func(file string, complete bool, err error), + ) error } func NewClient(inf kubernetes.Interface, cfg *rest.Config) Interface { @@ -98,13 +106,15 @@ func newRestClient(restConfig *rest.Config, gv schema.GroupVersion) (rest.Interf return rest.RESTClientFor(restConfig) } -func (r *Resource) WaitReady(ctx context.Context, obj Object) error { +func (r *Resource) WaitReady(ctx context.Context, obj Object, progressF func(Object)) error { if err := wait.PollImmediateInfiniteWithContext(ctx, time.Second, func(ctx context.Context) (bool, error) { fetched, err := r.Get(obj.GetNamespace(), obj.GetName()) if err != nil { return false, err } + fetched.GetObjectKind().SetGroupVersionKind(obj.GetObjectKind().GroupVersionKind()) + progressF(fetched.(Object)) readyable, ok := fetched.(interface{ GetStatusReady() bool }) if !ok { return false, fmt.Errorf("object is not readyable: %T", fetched) @@ -118,3 +128,20 @@ func (r *Resource) WaitReady(ctx context.Context, obj Object) error { return nil } + +func (r *Resource) Watch(ctx context.Context, namespace string, obj Object, opts *metav1.ListOptions) (watch.Interface, error) { + opts.Watch = true + if obj != nil && obj.GetName() != "" { + opts.ResourceVersion = obj.GetResourceVersion() + opts.FieldSelector = fields.OneTermEqualSelector("metadata.name", obj.GetName()).String() + } + + // NOTE: The r.Helper.Watch() method does not support passing a context, calling the code + // below instead (it was pulled from the Helper implementation). + w := r.RESTClient.Get(). + NamespaceIfScoped(namespace, r.NamespaceScoped). + Resource(r.Resource). + VersionedParams(opts, metav1.ParameterCodec) + + return w.Watch(ctx) +} diff --git a/kubectl/internal/cp/kubectl.go b/internal/client/cp/kubectl.go similarity index 100% rename from kubectl/internal/cp/kubectl.go rename to internal/client/cp/kubectl.go diff --git a/kubectl/internal/client/decode_encode.go b/internal/client/decode_encode.go similarity index 91% rename from kubectl/internal/client/decode_encode.go rename to internal/client/decode_encode.go index de554efa..a39129fd 100644 --- a/kubectl/internal/client/decode_encode.go +++ b/internal/client/decode_encode.go @@ -9,12 +9,13 @@ func Decode(data []byte) (Object, error) { decoder := scheme.Codecs.UniversalDeserializer() runtimeObject, gvk, err := decoder.Decode(data, nil, nil) + if gvk == nil { + return nil, nil + } if err != nil { return nil, err } - _ = gvk - return runtimeObject.(Object), nil } diff --git a/internal/client/notebook.go b/internal/client/notebook.go new file mode 100644 index 00000000..235d2d6f --- /dev/null +++ b/internal/client/notebook.go @@ -0,0 +1,86 @@ +package client + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + apiv1 "github.com/substratusai/substratus/api/v1" +) + +func PodForNotebook(nb *apiv1.Notebook) types.NamespacedName { + // TODO: Pull Pod info from status of Notebook. + return types.NamespacedName{ + Namespace: nb.GetNamespace(), + Name: nb.GetName() + "-notebook", + } +} + +func NotebookForObject(obj Object) (*apiv1.Notebook, error) { + var nb *apiv1.Notebook + + switch obj := obj.DeepCopyObject().(type) { + case *apiv1.Notebook: + nb = obj + + case *apiv1.Model: + nb = &apiv1.Notebook{ + ObjectMeta: metav1.ObjectMeta{ + Name: obj.Name + "-model", + Namespace: obj.Namespace, + }, + Spec: apiv1.NotebookSpec{ + Image: obj.Spec.Image, + Env: obj.Spec.Env, + Params: obj.Spec.Params, + Model: obj.Spec.Model, + Dataset: obj.Spec.Dataset, + Resources: obj.Spec.Resources, + }, + } + + case *apiv1.Server: + nb = &apiv1.Notebook{ + ObjectMeta: metav1.ObjectMeta{ + Name: obj.Name + "-server", + Namespace: obj.Namespace, + }, + Spec: apiv1.NotebookSpec{ + Image: obj.Spec.Image, + Env: obj.Spec.Env, + Params: obj.Spec.Params, + Model: &obj.Spec.Model, + Resources: obj.Spec.Resources, + }, + } + + case *apiv1.Dataset: + nb = &apiv1.Notebook{ + ObjectMeta: metav1.ObjectMeta{ + Name: obj.Name + "-dataset", + Namespace: obj.Namespace, + }, + Spec: apiv1.NotebookSpec{ + Image: obj.Spec.Image, + Env: obj.Spec.Env, + Params: obj.Spec.Params, + Resources: obj.Spec.Resources, + }, + } + + default: + return nil, fmt.Errorf("unknown object type: %T", obj) + } + + // "This field is managed by the API server and should not be changed by the user." + // https://kubernetes.io/docs/reference/using-api/server-side-apply/#field-management + nb.ObjectMeta.ManagedFields = nil + + nb.TypeMeta = metav1.TypeMeta{ + APIVersion: "substratus.ai/v1", + Kind: "Notebook", + } + + return nb, nil +} diff --git a/kubectl/internal/client/port_forward.go b/internal/client/port_forward.go similarity index 53% rename from kubectl/internal/client/port_forward.go rename to internal/client/port_forward.go index 9541fc38..71b45444 100644 --- a/kubectl/internal/client/port_forward.go +++ b/internal/client/port_forward.go @@ -6,26 +6,19 @@ import ( "io" "net/http" "net/url" - "os" "strings" - apiv1 "github.com/substratusai/substratus/api/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/portforward" "k8s.io/client-go/transport/spdy" ) -func podForNotebook(nb *apiv1.Notebook) types.NamespacedName { - // TODO: Pull Pod info from status of Notebook. - return types.NamespacedName{ - Namespace: nb.GetNamespace(), - Name: nb.GetName() + "-notebook", - } +type ForwardedPorts struct { + Local int + Pod int } -func (c *Client) PortForwardNotebook(ctx context.Context, verbose bool, nb *apiv1.Notebook, ready chan struct{}) error { - // TODO: Pull Pod info from status of Notebook. - podRef := podForNotebook(nb) +func (c *Client) PortForward(ctx context.Context, logger io.Writer, podRef types.NamespacedName, ports ForwardedPorts, ready chan struct{}) error { path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward", podRef.Namespace, podRef.Name) hostIP := strings.TrimLeft(c.Config.Host, "https://") @@ -35,18 +28,15 @@ func (c *Client) PortForwardNotebook(ctx context.Context, verbose bool, nb *apiv return err } - // TODO: Use an available local port, or allow it to be overridden. - localPort, podPort := 8888, 8888 - dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, http.MethodPost, &url.URL{Scheme: "https", Path: path, Host: hostIP}) var stdout, stderr io.Writer - if verbose { - stdout, stderr = os.Stdout, os.Stderr + if logger != nil { + stdout, stderr = logger, logger } else { stdout, stderr = io.Discard, io.Discard } - fw, err := portforward.New(dialer, []string{fmt.Sprintf("%d:%d", localPort, podPort)}, ctx.Done(), ready, stdout, stderr) + fw, err := portforward.New(dialer, []string{fmt.Sprintf("%d:%d", ports.Local, ports.Pod)}, ctx.Done(), ready, stdout, stderr) if err != nil { return err } diff --git a/kubectl/internal/client/sync.go b/internal/client/sync.go similarity index 82% rename from kubectl/internal/client/sync.go rename to internal/client/sync.go index 4c1f1bc8..9219a682 100644 --- a/kubectl/internal/client/sync.go +++ b/internal/client/sync.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "io" + "log" "net/http" "os" "path/filepath" @@ -19,14 +20,16 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/remotecommand" - "k8s.io/klog/v2" apiv1 "github.com/substratusai/substratus/api/v1" - "github.com/substratusai/substratus/kubectl/internal/cp" + "github.com/substratusai/substratus/internal/client/cp" ) -func (c *Client) SyncFilesFromNotebook(ctx context.Context, nb *apiv1.Notebook, localDir string) error { - podRef := podForNotebook(nb) +func (c *Client) SyncFilesFromNotebook(ctx context.Context, nb *apiv1.Notebook, localDir string, + logger io.Writer, + progressF func(file string, complete bool, err error), +) error { + podRef := PodForNotebook(nb) const containerName = "notebook" cacheDir, err := os.UserCacheDir() @@ -69,57 +72,64 @@ func (c *Client) SyncFilesFromNotebook(ctx context.Context, nb *apiv1.Notebook, go func() { defer func() { wg.Done() - klog.V(2).Info("File sync loop: Done.") + log.Print("File sync loop: Done.") }() - klog.V(2).Info("Reading events...") + log.Print("Reading events...") scanner := bufio.NewScanner(r) for scanner.Scan() { eventLine := scanner.Bytes() var event NBWatchEvent if err := json.Unmarshal(eventLine, &event); err != nil { - klog.Errorf("Failed to unmarshal nbevent: %w", err) + log.Printf("Non-json nbevent: %v", string(eventLine)) continue } - relPath, err := filepath.Rel("/content/src", event.Path) + relPath, err := filepath.Rel("/content", event.Path) if err != nil { - klog.Errorf("Failed to determine relative path: %w", err) + log.Printf("Failed to determine relative path: %v", err) continue } - localPath := filepath.Join(localDir, "src", relPath) + localPath := filepath.Join(localDir, relPath) // Possible: CREATE, REMOVE, WRITE, RENAME, CHMOD if event.Op == "WRITE" || event.Op == "CREATE" { // NOTE: A long-running port-forward might be more performant here. + progressF(event.Path, false, nil) if err := cp.FromPod(ctx, event.Path, localPath, podRef, containerName); err != nil { - klog.Errorf("Sync: failed to copy: %w", err) + log.Printf("Sync: failed to copy: %v", err) + progressF(event.Path, false, err) continue } + progressF(event.Path, true, nil) } else if event.Op == "REMOVE" || event.Op == "RENAME" { + progressF(event.Path, false, nil) if err := os.Remove(localPath); err != nil { - klog.Errorf("Sync: failed to remove: %w", err) + log.Printf("Sync: failed to remove: %v", err) + progressF(event.Path, false, err) continue } + progressF(event.Path, true, nil) } } if err := scanner.Err(); err != nil { - klog.Error("Error reading from buffer:", err) + log.Print("Error reading from buffer:", err) return } - klog.V(2).Info("Done reading events.") + log.Print("Done reading events.") }() - if err := c.exec(ctx, podRef, "/tmp/nbwatch", nil, w, os.Stderr); err != nil { + log.Print("Executing nbwatch...") + if err := c.exec(ctx, podRef, "/tmp/nbwatch", nil, w, logger); err != nil { w.Close() return fmt.Errorf("exec: nbwatch: %w", err) } - klog.V(2).Info("Waiting for file sync loop to finish...") + log.Print("Waiting for file sync loop to finish...") wg.Wait() - klog.V(2).Info("Done waiting for file sync loop to finish.") + log.Print("Done waiting for file sync loop to finish.") return nil } @@ -185,10 +195,10 @@ func getContainerTools(ctx context.Context, dir, targetOS string) error { } versionStr := strings.TrimSpace(string(version)) if versionStr == Version { - klog.V(1).Infof("Version (%q) matches for container-tools, skipping download.", Version) + log.Printf("Version (%q) matches for container-tools, skipping download.", Version) return nil } else { - klog.V(1).Infof("Version (%q) does not match version.txt: %q", Version, versionStr) + log.Printf("Version (%q) does not match version.txt: %q", Version, versionStr) } } @@ -216,7 +226,7 @@ func getContainerTools(ctx context.Context, dir, targetOS string) error { func getContainerToolsRelease(ctx context.Context, dir, targetOS, targetArch string) error { releaseURL := fmt.Sprintf("https://github.com/substratusai/substratus/releases/download/v%s/container-tools-%s-%s.tar.gz", Version, targetOS, targetArch) - klog.V(1).Infof("Downloading: %s", releaseURL) + log.Printf("Downloading: %s", releaseURL) req, err := http.NewRequestWithContext(ctx, "GET", releaseURL, nil) if err != nil { @@ -246,7 +256,7 @@ func getContainerToolsRelease(ctx context.Context, dir, targetOS, targetArch str } dest := filepath.Join(dir, hdr.Name) - klog.V(1).Infof("Writing %s", dest) + log.Printf("Writing %s", dest) f, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)) if err != nil { return fmt.Errorf("creating file: %w", err) diff --git a/kubectl/internal/client/upload.go b/internal/client/upload.go similarity index 84% rename from kubectl/internal/client/upload.go rename to internal/client/upload.go index df8e533e..23ff3d6a 100644 --- a/kubectl/internal/client/upload.go +++ b/internal/client/upload.go @@ -11,22 +11,20 @@ import ( "errors" "fmt" "io" + "log" "net/http" "os" "path/filepath" "strings" "time" - "k8s.io/utils/ptr" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/utils/ptr" apiv1 "github.com/substratusai/substratus/api/v1" - "k8s.io/klog/v2" ) var httpClient = &http.Client{} @@ -37,7 +35,7 @@ type Tarball struct { MD5Checksum string } -func PrepareImageTarball(ctx context.Context, buildPath string) (*Tarball, error) { +func PrepareImageTarball(ctx context.Context, buildPath string, progressF func(file string)) (*Tarball, error) { exists, err := fileExists(filepath.Join(buildPath, "Dockerfile")) if err != nil { return nil, fmt.Errorf("checking if Dockerfile exists: %w", err) @@ -46,13 +44,13 @@ func PrepareImageTarball(ctx context.Context, buildPath string) (*Tarball, error return nil, fmt.Errorf("path does not contain Dockerfile: %s", buildPath) } - tmpDir, err := os.MkdirTemp("/tmp", "substratus-kubectl-upload") + tmpDir, err := os.MkdirTemp("/tmp", "substratus-upload") if err != nil { return nil, fmt.Errorf("failed to create temp dir: %w", err) } tarPath := filepath.Join(tmpDir, "/archive.tar.gz") - err = tarGz(ctx, buildPath, tarPath) + err = tarGz(ctx, buildPath, tarPath, progressF) if err != nil { return nil, fmt.Errorf("failed to create a tar.gz of the directory: %w", err) } @@ -125,7 +123,7 @@ func (r *Resource) Apply(obj Object, force bool) error { return nil } -func (r *Resource) Upload(ctx context.Context, obj Object, tb *Tarball) error { +func (r *Resource) Upload(ctx context.Context, obj Object, tb *Tarball, progressF func(float64)) error { // NOTE: The r.Helper.WatchSingle() method does not support passing a context, calling the code // below instead (it was pulled from the Helper implementation). watcher, err := r.RESTClient.Get(). @@ -156,7 +154,7 @@ loop: if status.StoredMD5Checksum == tb.MD5Checksum { // This is an edge-case where the controller found a matching upload // that already existed in storage. - klog.V(1).Infof("upload already exists in storage with md5 checksum: %s, skipping upload", status.StoredMD5Checksum) + log.Printf("upload already exists in storage with md5 checksum: %s, skipping upload", status.StoredMD5Checksum) return nil } if status.SignedURL != "" && status.RequestID == spec.RequestID { @@ -179,7 +177,7 @@ loop: } } - if err := uploadTarball(tb, uploadURL); err != nil { + if err := uploadTarball(tb, uploadURL, progressF); err != nil { return fmt.Errorf("uploading tarball: %w", err) } @@ -208,7 +206,7 @@ func calculateMD5(path string) (string, error) { return fmt.Sprintf("%x", hash.Sum(nil)), nil } -func tarGz(ctx context.Context, src, dst string) error { +func tarGz(ctx context.Context, src, dst string, progressF func(string)) error { tarFile, err := os.Create(dst) if err != nil { return fmt.Errorf("failed to create tarFile: %w", err) @@ -221,9 +219,8 @@ func tarGz(ctx context.Context, src, dst string) error { tarWriter := tar.NewWriter(gzWriter) defer tarWriter.Close() - // TODO(bjb): #121 read .dockerignore if it exists, exclude those files - return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { - klog.V(4).Infof("Tarring: %v", path) + err = filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + log.Printf("Tarring: %v", path) if err := ctx.Err(); err != nil { return err } @@ -272,8 +269,15 @@ func tarGz(ctx context.Context, src, dst string) error { } } + progressF(path) + return nil }) + if err != nil { + return err + } + + return nil } func fileExists(filename string) (bool, error) { @@ -287,7 +291,21 @@ func fileExists(filename string) (bool, error) { return !info.IsDir(), nil } -func uploadTarball(tarball *Tarball, url string) error { +type progressReader struct { + totalRead int64 + total int64 + r io.Reader + f func(float64) +} + +func (r *progressReader) Read(p []byte) (int, error) { + n, err := r.r.Read(p) + r.totalRead += int64(n) + r.f(float64(r.totalRead) / float64(r.total)) + return n, err +} + +func uploadTarball(tarball *Tarball, url string, progressF func(float64)) error { data, err := hex.DecodeString(tarball.MD5Checksum) if err != nil { return fmt.Errorf("failed to decode hex checksum: %w", err) @@ -300,8 +318,17 @@ func uploadTarball(tarball *Tarball, url string) error { } defer file.Close() - klog.V(2).Infof("uploading tarball to: %s", url) - req, err := http.NewRequest(http.MethodPut, url, file) + stat, err := file.Stat() + if err != nil { + return fmt.Errorf("stat: %w", err) + } + + log.Printf("uploading tarball to: %s", url) + req, err := http.NewRequest(http.MethodPut, url, &progressReader{ + total: stat.Size(), + r: file, + f: progressF, + }) if err != nil { return fmt.Errorf("tar upload: %w", err) } @@ -319,6 +346,6 @@ func uploadTarball(tarball *Tarball, url string) error { if resp.StatusCode != http.StatusOK { return fmt.Errorf("unexpected response status: %d", resp.StatusCode) } - klog.V(1).Info("successfully uploaded tarball") + log.Print("successfully uploaded tarball") return nil } diff --git a/internal/cloud/cloud.go b/internal/cloud/cloud.go index 54c80653..d7dd6aef 100644 --- a/internal/cloud/cloud.go +++ b/internal/cloud/cloud.go @@ -8,10 +8,11 @@ import ( "cloud.google.com/go/compute/metadata" "github.com/go-playground/validator/v10" "github.com/sethvargo/go-envconfig" - apiv1 "github.com/substratusai/substratus/api/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1 "github.com/substratusai/substratus/api/v1" ) const CloudEnvVar = "CLOUD" diff --git a/internal/controller/build_reconciler.go b/internal/controller/build_reconciler.go index 07795603..796ed945 100644 --- a/internal/controller/build_reconciler.go +++ b/internal/controller/build_reconciler.go @@ -8,10 +8,6 @@ import ( "strings" "time" - apiv1 "github.com/substratusai/substratus/api/v1" - "github.com/substratusai/substratus/internal/cloud" - "github.com/substratusai/substratus/internal/resources" - "github.com/substratusai/substratus/internal/sci" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -23,6 +19,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + + apiv1 "github.com/substratusai/substratus/api/v1" + "github.com/substratusai/substratus/internal/cloud" + "github.com/substratusai/substratus/internal/resources" + "github.com/substratusai/substratus/internal/sci" ) const latestUploadPath = "uploads/latest.tar.gz" @@ -282,7 +283,8 @@ func (r *BuildReconciler) gitBuildJob(ctx context.Context, obj BuildableObject) // Disable compressed caching to decrease memory usage. // (See https://github.com/GoogleContainerTools/kaniko/blob/main/README.md#flag---compressed-caching) "--compressed-caching=false", - "--log-format=text", + "--log-format=color", + "--log-timestamp=false", } var initContainers []corev1.Container @@ -327,8 +329,10 @@ func (r *BuildReconciler) gitBuildJob(ctx context.Context, obj BuildableObject) corev1.Container{ Name: "gcp-workload-identity-readiness-check", Image: "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine", - Args: []string{"/bin/bash", "-c", - "curl -sS -H 'Metadata-Flavor: Google' 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token' --retry 30 --retry-connrefused --retry-max-time 60 --connect-timeout 3 --fail --retry-all-errors > /dev/null && exit 0 || echo 'Retry limit exceeded. Failed to wait for metadata server to be available. Check if the gke-metadata-server Pod in the kube-system namespace is healthy.' >&2; exit 1"}, + Args: []string{ + "/bin/bash", "-c", + "curl -sS -H 'Metadata-Flavor: Google' 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token' --retry 30 --retry-connrefused --retry-max-time 60 --connect-timeout 3 --fail --retry-all-errors > /dev/null && exit 0 || echo 'Retry limit exceeded. Failed to wait for metadata server to be available. Check if the gke-metadata-server Pod in the kube-system namespace is healthy.' >&2; exit 1", + }, }) } @@ -364,6 +368,10 @@ func (r *BuildReconciler) gitBuildJob(ctx context.Context, obj BuildableObject) Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: annotations, + Labels: map[string]string{ + strings.ToLower(r.Kind): obj.GetName(), + "role": "build", + }, }, Spec: corev1.PodSpec{ InitContainers: initContainers, @@ -408,7 +416,8 @@ func (r *BuildReconciler) storageBuildJob(ctx context.Context, obj BuildableObje // Disable compressed caching to decrease memory usage. // (See https://github.com/GoogleContainerTools/kaniko/blob/main/README.md#flag---compressed-caching) "--compressed-caching=false", - "--log-format=text", + "--log-format=color", + "--log-timestamp=false", } var initContainers []corev1.Container @@ -466,8 +475,10 @@ func (r *BuildReconciler) storageBuildJob(ctx context.Context, obj BuildableObje corev1.Container{ Name: "gcp-workload-identity-readiness-check", Image: "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine", - Args: []string{"/bin/bash", "-c", - "curl -sS -H 'Metadata-Flavor: Google' 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token' --retry 30 --retry-connrefused --retry-max-time 60 --connect-timeout 3 --fail --retry-all-errors > /dev/null && exit 0 || echo 'Retry limit exceeded. Failed to wait for metadata server to be available. Check if the gke-metadata-server Pod in the kube-system namespace is healthy.' >&2; exit 1"}, + Args: []string{ + "/bin/bash", "-c", + "curl -sS -H 'Metadata-Flavor: Google' 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token' --retry 30 --retry-connrefused --retry-max-time 60 --connect-timeout 3 --fail --retry-all-errors > /dev/null && exit 0 || echo 'Retry limit exceeded. Failed to wait for metadata server to be available. Check if the gke-metadata-server Pod in the kube-system namespace is healthy.' >&2; exit 1", + }, }) } @@ -487,6 +498,10 @@ func (r *BuildReconciler) storageBuildJob(ctx context.Context, obj BuildableObje Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: podAnnotations, + Labels: map[string]string{ + strings.ToLower(r.Kind): obj.GetName(), + "role": "build", + }, }, Spec: corev1.PodSpec{ InitContainers: initContainers, diff --git a/internal/controller/dataset_controller.go b/internal/controller/dataset_controller.go index 3f44dd6c..514bbd08 100644 --- a/internal/controller/dataset_controller.go +++ b/internal/controller/dataset_controller.go @@ -103,25 +103,38 @@ func (r *DatasetReconciler) reconcileData(ctx context.Context, dataset *apiv1.Da return result{}, nil } - dataset.Status.Ready = false - meta.SetStatusCondition(dataset.GetConditions(), metav1.Condition{ - Type: apiv1.ConditionLoaded, - Status: metav1.ConditionFalse, - Reason: apiv1.ReasonJobNotComplete, - ObservedGeneration: dataset.Generation, - Message: "Waiting for data loader Job to complete", - }) if err := r.Status().Update(ctx, dataset); err != nil { return result{}, fmt.Errorf("updating status: %w", err) } - if result, err := reconcileJob(ctx, r.Client, loadJob, apiv1.ConditionLoaded); !result.success { - return result, err + jobResult, err := reconcileJob(ctx, r.Client, loadJob) + if !jobResult.success { + dataset.Status.Ready = false + if !jobResult.failure { + meta.SetStatusCondition(dataset.GetConditions(), metav1.Condition{ + Type: apiv1.ConditionComplete, + Status: metav1.ConditionFalse, + Reason: apiv1.ReasonJobNotComplete, + ObservedGeneration: dataset.Generation, + Message: "Waiting for data loader Job to complete", + }) + } else { + meta.SetStatusCondition(dataset.GetConditions(), metav1.Condition{ + Type: apiv1.ConditionComplete, + Status: metav1.ConditionFalse, + Reason: apiv1.ReasonJobFailed, + ObservedGeneration: dataset.Generation, + }) + } + if err := r.Status().Update(ctx, dataset); err != nil { + return result{}, fmt.Errorf("updating status: %w", err) + } + return jobResult, err } dataset.Status.Ready = true meta.SetStatusCondition(dataset.GetConditions(), metav1.Condition{ - Type: apiv1.ConditionLoaded, + Type: apiv1.ConditionComplete, Status: metav1.ConditionTrue, Reason: apiv1.ReasonJobComplete, ObservedGeneration: dataset.Generation, @@ -146,11 +159,16 @@ func (r *DatasetReconciler) loadJob(ctx context.Context, dataset *apiv1.Dataset) Namespace: dataset.Namespace, }, Spec: batchv1.JobSpec{ + BackoffLimit: ptr.To(int32(2)), // TotalRetries = BackoffLimit + 1 Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "kubectl.kubernetes.io/default-container": containerName, }, + Labels: map[string]string{ + "dataset": dataset.Name, + "role": "run", + }, }, Spec: corev1.PodSpec{ SecurityContext: &corev1.PodSecurityContext{ @@ -176,10 +194,9 @@ func (r *DatasetReconciler) loadJob(ctx context.Context, dataset *apiv1.Dataset) } if err := r.Cloud.MountBucket(&job.Spec.Template.ObjectMeta, &job.Spec.Template.Spec, dataset, cloud.MountBucketConfig{ - Name: "dataset", + Name: "artifacts", Mounts: []cloud.BucketMount{ - {BucketSubdir: "data", ContentSubdir: "data"}, - {BucketSubdir: "logs", ContentSubdir: "logs"}, + {BucketSubdir: "artifacts", ContentSubdir: "artifacts"}, }, Container: containerName, ReadOnly: false, diff --git a/internal/controller/dataset_controller_test.go b/internal/controller/dataset_controller_test.go index 2e7ad5f6..0ecafd49 100644 --- a/internal/controller/dataset_controller_test.go +++ b/internal/controller/dataset_controller_test.go @@ -6,7 +6,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - apiv1 "github.com/substratusai/substratus/api/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -14,6 +13,8 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1 "github.com/substratusai/substratus/api/v1" ) func TestDataset(t *testing.T) { @@ -68,9 +69,8 @@ func testDatasetLoad(t *testing.T, dataset *apiv1.Dataset) { require.EventuallyWithT(t, func(t *assert.CollectT) { err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dataset), dataset) assert.NoError(t, err, "getting the dataset") - assert.True(t, meta.IsStatusConditionTrue(dataset.Status.Conditions, apiv1.ConditionLoaded)) + assert.True(t, meta.IsStatusConditionTrue(dataset.Status.Conditions, apiv1.ConditionComplete)) assert.True(t, dataset.Status.Ready) }, timeout, interval, "waiting for the dataset to be ready") require.Contains(t, dataset.Status.Artifacts.URL, "gs://test-artifact-bucket") - } diff --git a/internal/controller/main_test.go b/internal/controller/main_test.go index b00cddec..c86c45d4 100644 --- a/internal/controller/main_test.go +++ b/internal/controller/main_test.go @@ -11,26 +11,24 @@ import ( "testing" "time" - ctrl "sigs.k8s.io/controller-runtime" - + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" apiv1 "github.com/substratusai/substratus/api/v1" "github.com/substratusai/substratus/internal/cloud" "github.com/substratusai/substratus/internal/controller" "github.com/substratusai/substratus/internal/sci" - //+kubebuilder:scaffold:imports ) const ( @@ -46,10 +44,10 @@ var ( ) func TestMain(m *testing.M) { - //var buf bytes.Buffer + // var buf bytes.Buffer logf.SetLogger(zap.New( zap.UseDevMode(true), - //zap.WriteTo(&buf), + // zap.WriteTo(&buf), )) ctx, cancel = context.WithCancel(context.TODO()) @@ -87,8 +85,8 @@ func TestMain(m *testing.M) { sciClient := &sci.FakeSCIControllerClient{} - //runtimeMgr, err := controller.NewRuntimeManager(controller.GPUTypeNvidiaL4) - //requireNoError(err) + // runtimeMgr, err := controller.NewRuntimeManager(controller.GPUTypeNvidiaL4) + // requireNoError(err) err = (&controller.ModelReconciler{ Client: mgr.GetClient(), @@ -247,6 +245,12 @@ func testParamsConfigMap(t *testing.T, obj testObject, kind string, content stri func fakeJobComplete(t *testing.T, job *batchv1.Job) { updated := job.DeepCopy() updated.Status.Succeeded = 1 + updated.Status.Conditions = []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }, + } require.NoError(t, k8sClient.Status().Patch(ctx, updated, client.MergeFrom(job)), "patching the job with completed count") } diff --git a/internal/controller/manager.go b/internal/controller/manager.go index 70b745cd..80882cd8 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -4,17 +4,18 @@ import ( "context" "fmt" - apiv1 "github.com/substratusai/substratus/api/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" + + apiv1 "github.com/substratusai/substratus/api/v1" ) const ( notebookModelIndex = "spec.model.name" notebookDatasetIndex = "spec.dataset.name" - modelBaseModelIndex = "spec.baseModel.name" - modelTrainingDatasetIndex = "spec.trainingDataset.name" + modelModelIndex = "spec.model.name" + modelDatasetIndex = "spec.dataset.name" modelServerModelIndex = "spec.model.name" ) @@ -40,22 +41,22 @@ func SetupIndexes(mgr manager.Manager) error { return fmt.Errorf("notebook: %w", err) } - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &apiv1.Model{}, modelBaseModelIndex, func(rawObj client.Object) []string { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &apiv1.Model{}, modelModelIndex, func(rawObj client.Object) []string { model := rawObj.(*apiv1.Model) - if model.Spec.BaseModel == nil { + if model.Spec.Model == nil { return []string{} } - return []string{model.Spec.BaseModel.Name} + return []string{model.Spec.Model.Name} }); err != nil { return fmt.Errorf("model: %w", err) } - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &apiv1.Model{}, modelTrainingDatasetIndex, func(rawObj client.Object) []string { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &apiv1.Model{}, modelDatasetIndex, func(rawObj client.Object) []string { model := rawObj.(*apiv1.Model) - if model.Spec.TrainingDataset == nil { + if model.Spec.Dataset == nil { return []string{} } - return []string{model.Spec.TrainingDataset.Name} + return []string{model.Spec.Dataset.Name} }); err != nil { return fmt.Errorf("model: %w", err) } diff --git a/internal/controller/model_controller.go b/internal/controller/model_controller.go index f373a537..6c6f7c5c 100644 --- a/internal/controller/model_controller.go +++ b/internal/controller/model_controller.go @@ -90,14 +90,14 @@ func (r *ModelReconciler) reconcileModel(ctx context.Context, model *apiv1.Model } var baseModel *apiv1.Model - if model.Spec.BaseModel != nil { + if model.Spec.Model != nil { baseModel = &apiv1.Model{} - if err := r.Client.Get(ctx, types.NamespacedName{Namespace: model.Namespace, Name: model.Spec.BaseModel.Name}, baseModel); err != nil { + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: model.Namespace, Name: model.Spec.Model.Name}, baseModel); err != nil { if apierrors.IsNotFound(err) { // Update this Model's status. model.Status.Ready = false meta.SetStatusCondition(&model.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionModelled, + Type: apiv1.ConditionComplete, Status: metav1.ConditionFalse, Reason: apiv1.ReasonBaseModelNotFound, ObservedGeneration: model.Generation, @@ -116,7 +116,7 @@ func (r *ModelReconciler) reconcileModel(ctx context.Context, model *apiv1.Model // Update this Model's status. model.Status.Ready = false meta.SetStatusCondition(&model.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionModelled, + Type: apiv1.ConditionComplete, Status: metav1.ConditionFalse, Reason: apiv1.ReasonBaseModelNotReady, ObservedGeneration: model.Generation, @@ -131,14 +131,14 @@ func (r *ModelReconciler) reconcileModel(ctx context.Context, model *apiv1.Model } var dataset *apiv1.Dataset - if model.Spec.TrainingDataset != nil { + if model.Spec.Dataset != nil { dataset = &apiv1.Dataset{} - if err := r.Client.Get(ctx, types.NamespacedName{Namespace: model.Namespace, Name: model.Spec.TrainingDataset.Name}, dataset); err != nil { + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: model.Namespace, Name: model.Spec.Dataset.Name}, dataset); err != nil { if apierrors.IsNotFound(err) { // Update this Model's status. model.Status.Ready = false meta.SetStatusCondition(&model.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionModelled, + Type: apiv1.ConditionComplete, Status: metav1.ConditionFalse, Reason: apiv1.ReasonDatasetNotFound, ObservedGeneration: model.Generation, @@ -157,7 +157,7 @@ func (r *ModelReconciler) reconcileModel(ctx context.Context, model *apiv1.Model // Update this Model's status. model.Status.Ready = false meta.SetStatusCondition(&model.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionModelled, + Type: apiv1.ConditionComplete, Status: metav1.ConditionFalse, Reason: apiv1.ReasonDatasetNotReady, ObservedGeneration: model.Generation, @@ -178,25 +178,34 @@ func (r *ModelReconciler) reconcileModel(ctx context.Context, model *apiv1.Model return result{}, nil } - model.Status.Ready = false - meta.SetStatusCondition(model.GetConditions(), metav1.Condition{ - Type: apiv1.ConditionModelled, - Status: metav1.ConditionFalse, - Reason: apiv1.ReasonJobNotComplete, - ObservedGeneration: model.Generation, - Message: "Waiting for modeller Job to complete", - }) - if err := r.Status().Update(ctx, model); err != nil { - return result{}, fmt.Errorf("updating status: %w", err) - } - - if result, err := reconcileJob(ctx, r.Client, modellerJob, apiv1.ConditionModelled); !result.success { - return result, err + jobResult, err := reconcileJob(ctx, r.Client, modellerJob) + if !jobResult.success { + model.Status.Ready = false + if !jobResult.failure { + meta.SetStatusCondition(model.GetConditions(), metav1.Condition{ + Type: apiv1.ConditionComplete, + Status: metav1.ConditionFalse, + Reason: apiv1.ReasonJobNotComplete, + ObservedGeneration: model.Generation, + Message: "Waiting for modeller Job to complete", + }) + } else { + meta.SetStatusCondition(model.GetConditions(), metav1.Condition{ + Type: apiv1.ConditionComplete, + Status: metav1.ConditionFalse, + Reason: apiv1.ReasonJobFailed, + ObservedGeneration: model.Generation, + }) + } + if err := r.Status().Update(ctx, model); err != nil { + return result{}, fmt.Errorf("updating status: %w", err) + } + return jobResult, err } model.Status.Ready = true meta.SetStatusCondition(model.GetConditions(), metav1.Condition{ - Type: apiv1.ConditionModelled, + Type: apiv1.ConditionComplete, Status: metav1.ConditionTrue, Reason: apiv1.ReasonJobComplete, ObservedGeneration: model.Generation, @@ -230,7 +239,7 @@ func (r *ModelReconciler) findModelsForBaseModel(ctx context.Context, obj client var models apiv1.ModelList if err := r.List(ctx, &models, - client.MatchingFields{modelBaseModelIndex: model.Name}, + client.MatchingFields{modelModelIndex: model.Name}, client.InNamespace(obj.GetNamespace()), ); err != nil { log.Log.Error(err, "unable to list models for base model") @@ -254,7 +263,7 @@ func (r *ModelReconciler) findModelsForDataset(ctx context.Context, obj client.O var models apiv1.ModelList if err := r.List(ctx, &models, - client.MatchingFields{modelTrainingDatasetIndex: dataset.Name}, + client.MatchingFields{modelDatasetIndex: dataset.Name}, client.InNamespace(obj.GetNamespace()), ); err != nil { log.Log.Error(err, "unable to list models for dataset") @@ -282,6 +291,17 @@ func (r *ModelReconciler) modellerJob(ctx context.Context, model, baseModel *api return nil, fmt.Errorf("resolving env: %w", err) } + // Don't retry expensive Jobs by default. + var backoffLimit int32 + if model.Spec.Resources != nil && + model.Spec.Resources.CPU <= 3 && + model.Spec.Resources.GPU != nil && + model.Spec.Resources.GPU.Count == 0 { + // If the Job is not super expensive (No GPUs, low CPUs), then assume + // it is an import Job and up the retry count. + backoffLimit = 2 // 2 = 3 retries + } + const containerName = "model" job = &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ @@ -290,12 +310,17 @@ func (r *ModelReconciler) modellerJob(ctx context.Context, model, baseModel *api Namespace: model.Namespace, }, Spec: batchv1.JobSpec{ - BackoffLimit: ptr.To(int32(1)), + // TODO: Allow for configurable retries for Jobs that import models... + BackoffLimit: ptr.To(backoffLimit), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "kubectl.kubernetes.io/default-container": containerName, }, + Labels: map[string]string{ + "model": model.Name, + "role": "run", + }, }, Spec: corev1.PodSpec{ SecurityContext: &corev1.PodSecurityContext{ @@ -321,10 +346,9 @@ func (r *ModelReconciler) modellerJob(ctx context.Context, model, baseModel *api } if err := r.Cloud.MountBucket(&job.Spec.Template.ObjectMeta, &job.Spec.Template.Spec, model, cloud.MountBucketConfig{ - Name: "model", + Name: "artifacts", Mounts: []cloud.BucketMount{ - {BucketSubdir: "model", ContentSubdir: "model"}, - {BucketSubdir: "logs", ContentSubdir: "logs"}, + {BucketSubdir: "artifacts", ContentSubdir: "artifacts"}, }, Container: containerName, ReadOnly: false, @@ -336,7 +360,7 @@ func (r *ModelReconciler) modellerJob(ctx context.Context, model, baseModel *api if err := r.Cloud.MountBucket(&job.Spec.Template.ObjectMeta, &job.Spec.Template.Spec, dataset, cloud.MountBucketConfig{ Name: "dataset", Mounts: []cloud.BucketMount{ - {BucketSubdir: "data", ContentSubdir: "data"}, + {BucketSubdir: "artifacts", ContentSubdir: "data"}, }, Container: containerName, ReadOnly: true, @@ -347,9 +371,9 @@ func (r *ModelReconciler) modellerJob(ctx context.Context, model, baseModel *api if baseModel != nil { if err := r.Cloud.MountBucket(&job.Spec.Template.ObjectMeta, &job.Spec.Template.Spec, baseModel, cloud.MountBucketConfig{ - Name: "basemodel", + Name: "model", Mounts: []cloud.BucketMount{ - {BucketSubdir: "model", ContentSubdir: "saved-model"}, + {BucketSubdir: "artifacts", ContentSubdir: "model"}, }, Container: containerName, ReadOnly: true, diff --git a/internal/controller/model_controller_test.go b/internal/controller/model_controller_test.go index 930084ed..d478a7d3 100644 --- a/internal/controller/model_controller_test.go +++ b/internal/controller/model_controller_test.go @@ -6,7 +6,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - apiv1 "github.com/substratusai/substratus/api/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -14,6 +13,8 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" + + apiv1 "github.com/substratusai/substratus/api/v1" ) func TestModelLoaderFromGit(t *testing.T) { @@ -58,7 +59,7 @@ func testModelLoad(t *testing.T, model *apiv1.Model) { require.EventuallyWithT(t, func(t *assert.CollectT) { err := k8sClient.Get(ctx, types.NamespacedName{Namespace: model.GetNamespace(), Name: model.GetName()}, model) assert.NoError(t, err, "getting model") - assert.True(t, meta.IsStatusConditionTrue(model.Status.Conditions, apiv1.ConditionModelled)) + assert.True(t, meta.IsStatusConditionTrue(model.Status.Conditions, apiv1.ConditionComplete)) assert.True(t, model.Status.Ready) }, timeout, interval, "waiting for the model to be ready") require.Contains(t, model.Status.Artifacts.URL, "gs://test-artifact-bucket") @@ -109,10 +110,10 @@ func TestModelTrainerFromGit(t *testing.T) { URL: "https://test.com/test/test", }, }, - BaseModel: &apiv1.ObjectRef{ + Model: &apiv1.ObjectRef{ Name: baseModel.Name, }, - TrainingDataset: &apiv1.ObjectRef{ + Dataset: &apiv1.ObjectRef{ Name: dataset.Name, }, }, @@ -151,7 +152,7 @@ func testModelTrain(t *testing.T, model *apiv1.Model) { require.EventuallyWithT(t, func(t *assert.CollectT) { err := k8sClient.Get(ctx, types.NamespacedName{Namespace: model.GetNamespace(), Name: model.GetName()}, model) assert.NoError(t, err, "getting model") - assert.True(t, meta.IsStatusConditionTrue(model.Status.Conditions, apiv1.ConditionModelled)) + assert.True(t, meta.IsStatusConditionTrue(model.Status.Conditions, apiv1.ConditionComplete)) assert.True(t, model.Status.Ready) }, timeout, interval, "waiting for the model to be ready") require.Contains(t, model.Status.Artifacts.URL, "gs://test-artifact-bucket") diff --git a/internal/controller/notebook_controller.go b/internal/controller/notebook_controller.go index b9ca6479..4736becd 100644 --- a/internal/controller/notebook_controller.go +++ b/internal/controller/notebook_controller.go @@ -134,7 +134,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap if notebook.IsSuspended() { notebook.Status.Ready = false meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionDeployed, + Type: apiv1.ConditionServing, Status: metav1.ConditionFalse, Reason: apiv1.ReasonSuspended, ObservedGeneration: notebook.Generation, @@ -167,7 +167,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap } var model *apiv1.Model - if notebook.Spec.Model != nil { + if notebook.Spec.Model != nil && notebook.Spec.Model.Name != "" { model = &apiv1.Model{} if err := r.Get(ctx, client.ObjectKey{Name: notebook.Spec.Model.Name, Namespace: notebook.Namespace}, model); err != nil { if apierrors.IsNotFound(err) { @@ -175,7 +175,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap // Update this Model's status. notebook.Status.Ready = false meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionDeployed, + Type: apiv1.ConditionServing, Status: metav1.ConditionFalse, Reason: apiv1.ReasonModelNotFound, ObservedGeneration: notebook.Generation, @@ -195,7 +195,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap notebook.Status.Ready = false meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionDeployed, + Type: apiv1.ConditionServing, Status: metav1.ConditionFalse, Reason: apiv1.ReasonModelNotReady, ObservedGeneration: notebook.Generation, @@ -210,14 +210,14 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap } var dataset *apiv1.Dataset - if notebook.Spec.Dataset != nil { + if notebook.Spec.Dataset != nil && notebook.Spec.Dataset.Name != "" { dataset = &apiv1.Dataset{} if err := r.Get(ctx, client.ObjectKey{Name: notebook.Spec.Dataset.Name, Namespace: notebook.Namespace}, dataset); err != nil { if apierrors.IsNotFound(err) { log.Error(err, "Dataset not found") notebook.Status.Ready = false meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionDeployed, + Type: apiv1.ConditionServing, Status: metav1.ConditionFalse, Reason: apiv1.ReasonDatasetNotFound, ObservedGeneration: notebook.Generation, @@ -236,7 +236,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap log.Info("Dataset not ready", "dataset", dataset.Name) notebook.Status.Ready = false meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionDeployed, + Type: apiv1.ConditionServing, Status: metav1.ConditionFalse, Reason: apiv1.ReasonDatasetNotReady, ObservedGeneration: notebook.Generation, @@ -288,7 +288,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap if isPodReady(pod) { notebook.Status.Ready = true meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionDeployed, + Type: apiv1.ConditionServing, Status: metav1.ConditionTrue, Reason: apiv1.ReasonPodReady, ObservedGeneration: notebook.Generation, @@ -296,7 +296,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap } else { notebook.Status.Ready = false meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionDeployed, + Type: apiv1.ConditionServing, Status: metav1.ConditionFalse, Reason: apiv1.ReasonPodNotReady, ObservedGeneration: notebook.Generation, @@ -317,10 +317,22 @@ func nbPodName(nb *apiv1.Notebook) string { func (r *NotebookReconciler) notebookPod(notebook *apiv1.Notebook, model *apiv1.Model, dataset *apiv1.Dataset) (*corev1.Pod, error) { const containerName = "notebook" - envVars, err := resolveEnv(notebook.Spec.Env) + cmd := notebook.Spec.Command + if cmd == nil { + cmd = []string{ + "jupyter", "lab", + "--allow-root", + "--ip=0.0.0.0", + "--NotebookApp.token=$(NOTEBOOK_TOKEN)", + "--notebook-dir=/content", + } + } + + env, err := resolveEnv(notebook.Spec.Env) if err != nil { return nil, fmt.Errorf("resolving env: %w", err) } + env = append(env, corev1.EnvVar{Name: "NOTEBOOK_TOKEN", Value: "default"}) pod := &corev1.Pod{ TypeMeta: metav1.TypeMeta{ @@ -333,6 +345,10 @@ func (r *NotebookReconciler) notebookPod(notebook *apiv1.Notebook, model *apiv1. Annotations: map[string]string{ "kubectl.kubernetes.io/default-container": containerName, }, + Labels: map[string]string{ + "notebook": notebook.Name, + "role": "run", + }, }, Spec: corev1.PodSpec{ //SecurityContext: &corev1.PodSecurityContext{ @@ -345,15 +361,16 @@ func (r *NotebookReconciler) notebookPod(notebook *apiv1.Notebook, model *apiv1. { Name: containerName, Image: notebook.GetImage(), - Command: notebook.Spec.Command, - //WorkingDir: "/home/jovyan", + Command: cmd, + + // WorkingDir: "/home/jovyan", Ports: []corev1.ContainerPort{ { Name: "notebook", ContainerPort: 8888, }, }, - Env: envVars, + Env: env, // TODO: GPUs ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ @@ -392,8 +409,7 @@ func (r *NotebookReconciler) notebookPod(notebook *apiv1.Notebook, model *apiv1. if err := r.Cloud.MountBucket(&pod.ObjectMeta, &pod.Spec, dataset, cloud.MountBucketConfig{ Name: "dataset", Mounts: []cloud.BucketMount{ - {BucketSubdir: "data", ContentSubdir: "data"}, - {BucketSubdir: "logs", ContentSubdir: "data-logs"}, + {BucketSubdir: "artifacts", ContentSubdir: "data"}, }, Container: containerName, ReadOnly: true, @@ -404,10 +420,9 @@ func (r *NotebookReconciler) notebookPod(notebook *apiv1.Notebook, model *apiv1. if model != nil { if err := r.Cloud.MountBucket(&pod.ObjectMeta, &pod.Spec, model, cloud.MountBucketConfig{ - Name: "basemodel", + Name: "model", Mounts: []cloud.BucketMount{ - {BucketSubdir: "model", ContentSubdir: "saved-model"}, - {BucketSubdir: "logs", ContentSubdir: "saved-model-logs"}, + {BucketSubdir: "artifacts", ContentSubdir: "model"}, }, Container: containerName, ReadOnly: true, @@ -416,6 +431,16 @@ func (r *NotebookReconciler) notebookPod(notebook *apiv1.Notebook, model *apiv1. } } + // Mounts specific to this Notebook. + if err := r.Cloud.MountBucket(&pod.ObjectMeta, &pod.Spec, notebook, cloud.MountBucketConfig{ + Name: "artifacts", + Mounts: []cloud.BucketMount{{BucketSubdir: "artifacts", ContentSubdir: "artifacts"}}, + Container: containerName, + ReadOnly: false, + }); err != nil { + return nil, fmt.Errorf("mounting notebook: %w", err) + } + if err := ctrl.SetControllerReference(notebook, pod, r.Scheme); err != nil { return nil, fmt.Errorf("failed to set controller reference: %w", err) } diff --git a/internal/controller/notebook_controller_test.go b/internal/controller/notebook_controller_test.go index 79f43f72..0b56825c 100644 --- a/internal/controller/notebook_controller_test.go +++ b/internal/controller/notebook_controller_test.go @@ -6,7 +6,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - apiv1 "github.com/substratusai/substratus/api/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,6 +13,8 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1 "github.com/substratusai/substratus/api/v1" ) func TestNotebookFromGitWithModelOnly(t *testing.T) { @@ -75,7 +76,7 @@ func TestNotebookFromGitWithModelOnly(t *testing.T) { require.EventuallyWithT(t, func(t *assert.CollectT) { err := k8sClient.Get(ctx, client.ObjectKeyFromObject(notebook), notebook) assert.NoError(t, err, "getting the notebook") - assert.True(t, meta.IsStatusConditionTrue(notebook.Status.Conditions, apiv1.ConditionDeployed)) + assert.True(t, meta.IsStatusConditionTrue(notebook.Status.Conditions, apiv1.ConditionServing)) assert.True(t, notebook.Status.Ready) }, timeout, interval, "waiting for the notebook to be ready") } diff --git a/internal/controller/server_controller.go b/internal/controller/server_controller.go index 0285a3ae..b32b70b7 100644 --- a/internal/controller/server_controller.go +++ b/internal/controller/server_controller.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -19,7 +20,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/go-logr/logr" apiv1 "github.com/substratusai/substratus/api/v1" "github.com/substratusai/substratus/internal/cloud" "github.com/substratusai/substratus/internal/resources" @@ -101,7 +101,6 @@ func (r *ServerReconciler) findServersForModel(ctx context.Context, obj client.O reqs := []reconcile.Request{} for _, svr := range servers.Items { - reqs = append(reqs, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: svr.Name, @@ -185,7 +184,7 @@ func (r *ServerReconciler) serverDeployment(server *apiv1.Server, model *apiv1.M if err := r.Cloud.MountBucket(&deploy.Spec.Template.ObjectMeta, &deploy.Spec.Template.Spec, model, cloud.MountBucketConfig{ Name: "model", Mounts: []cloud.BucketMount{ - {BucketSubdir: "model", ContentSubdir: "saved-model"}, + {BucketSubdir: "artifacts", ContentSubdir: "model"}, }, Container: containerName, ReadOnly: true, @@ -214,7 +213,7 @@ func (r *ServerReconciler) reconcileServer(ctx context.Context, server *apiv1.Se // Update this Model's status. server.Status.Ready = false meta.SetStatusCondition(&server.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionDeployed, + Type: apiv1.ConditionServing, Status: metav1.ConditionFalse, Reason: apiv1.ReasonModelNotFound, ObservedGeneration: server.Generation, @@ -234,7 +233,7 @@ func (r *ServerReconciler) reconcileServer(ctx context.Context, server *apiv1.Se server.Status.Ready = false meta.SetStatusCondition(&server.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionDeployed, + Type: apiv1.ConditionServing, Status: metav1.ConditionFalse, Reason: apiv1.ReasonModelNotReady, ObservedGeneration: server.Generation, @@ -281,7 +280,7 @@ func (r *ServerReconciler) reconcileServer(ctx context.Context, server *apiv1.Se if deploy.Status.ReadyReplicas == 0 { server.Status.Ready = false meta.SetStatusCondition(&server.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionDeployed, + Type: apiv1.ConditionServing, Status: metav1.ConditionFalse, Reason: apiv1.ReasonDeploymentNotReady, ObservedGeneration: server.Generation, @@ -289,7 +288,7 @@ func (r *ServerReconciler) reconcileServer(ctx context.Context, server *apiv1.Se } else { server.Status.Ready = true meta.SetStatusCondition(&server.Status.Conditions, metav1.Condition{ - Type: apiv1.ConditionDeployed, + Type: apiv1.ConditionServing, Status: metav1.ConditionTrue, Reason: apiv1.ReasonDeploymentReady, ObservedGeneration: server.Generation, @@ -336,7 +335,7 @@ func (r *ServerReconciler) serverService(server *apiv1.Server, model *apiv1.Mode } func withServerSelector(server *apiv1.Server, labels map[string]string) map[string]string { - labels["component"] = "server" + labels["role"] = "run" labels["server"] = server.Name return labels } diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 0b3fae65..9c694fef 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -17,9 +17,10 @@ import ( type result struct { ctrl.Result success bool + failure bool } -func reconcileJob(ctx context.Context, c client.Client, job *batchv1.Job, condition string) (result, error) { +func reconcileJob(ctx context.Context, c client.Client, job *batchv1.Job) (result, error) { if err := c.Create(ctx, job); client.IgnoreAlreadyExists(err) != nil { return result{}, fmt.Errorf("creating Job: %w", err) } @@ -27,12 +28,24 @@ func reconcileJob(ctx context.Context, c client.Client, job *batchv1.Job, condit if err := c.Get(ctx, client.ObjectKeyFromObject(job), job); err != nil { return result{}, fmt.Errorf("geting Job: %w", err) } - if job.Status.Succeeded < 1 { - // Allow Job watch to requeue. - return result{}, nil - } - return result{success: true}, nil + complete, failed := jobResult(job) + + return result{success: complete, failure: failed}, nil +} + +func jobResult(job *batchv1.Job) (complete bool, failed bool) { + for _, c := range job.Status.Conditions { + if c.Type == batchv1.JobComplete && c.Status == corev1.ConditionTrue { + complete = true + return + } + if c.Type == batchv1.JobFailed && c.Status == corev1.ConditionTrue { + failed = true + return + } + } + return } func isPodReady(pod *corev1.Pod) bool { diff --git a/internal/tui/common.go b/internal/tui/common.go new file mode 100644 index 00000000..141a637c --- /dev/null +++ b/internal/tui/common.go @@ -0,0 +1,304 @@ +package tui + +import ( + "context" + "fmt" + "log" + "os" + "strconv" + "strings" + + tea "github.com/charmbracelet/bubbletea" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + + apiv1 "github.com/substratusai/substratus/api/v1" + "github.com/substratusai/substratus/internal/cli/utils" + "github.com/substratusai/substratus/internal/client" +) + +var ( + P *tea.Program + LogFile *os.File +) + +func init() { + // Log to a file. Useful in debugging since you can't really log to stdout. + var err error + LogFile, err = tea.LogToFile("/tmp/sub.log", "") + if err != nil { + panic(err) + } +} + +type Namespace struct { + Specified string + Contextual string +} + +func (n Namespace) Set(obj client.Object) { + if n.Specified != "" { + obj.SetNamespace(n.Specified) + } else if obj.GetNamespace() == "" { + ns := "default" + if n.Contextual != "" { + ns = n.Contextual + } + obj.SetNamespace(ns) + } +} + +type status int + +const ( + notStarted = status(0) + inProgress = status(1) + completed = status(2) +) + +type localURLMsg string + +type ( + tarballCompleteMsg *client.Tarball + fileTarredMsg string +) + +func prepareTarballCmd(ctx context.Context, dir string) tea.Cmd { + return func() tea.Msg { + log.Println("Preparing tarball") + tarball, err := client.PrepareImageTarball(ctx, dir, func(file string) { + log.Println("tarred", file) + P.Send(fileTarredMsg(file)) + }) + if err != nil { + log.Println("Error", err) + return fmt.Errorf("preparing tarball: %w", err) + } + return tarballCompleteMsg(tarball) + } +} + +type ( + tarballUploadedMsg struct { + client.Object + } + uploadTarballProgressMsg float64 +) + +func uploadTarballCmd(ctx context.Context, res *client.Resource, obj client.Object, tarball *client.Tarball) tea.Cmd { + return func() tea.Msg { + log.Println("Uploading tarball") + err := res.Upload(ctx, obj, tarball, func(percentage float64) { + log.Printf("Upload percentage: %v", percentage) + P.Send(uploadTarballProgressMsg(percentage)) + }) + if err != nil { + log.Println("Upload failed", err) + return fmt.Errorf("uploading: %w", err) + } + log.Println("Upload completed") + return tarballUploadedMsg{Object: obj} + } +} + +func specifyUpload(obj client.Object, tarball *client.Tarball) error { + if err := client.ClearImage(obj); err != nil { + return fmt.Errorf("clearing image in spec: %w", err) + } + if err := client.SetUploadContainerSpec(obj, tarball, utils.NewUUID()); err != nil { + return fmt.Errorf("setting upload in spec: %w", err) + } + return nil +} + +type appliedWithUploadMsg struct { + client.Object +} + +func applyWithUploadCmd(ctx context.Context, res *client.Resource, obj client.Object, tarball *client.Tarball) tea.Cmd { + return func() tea.Msg { + if err := specifyUpload(obj, tarball); err != nil { + return fmt.Errorf("specifying upload: %w", err) + } + if err := res.Apply(obj, true); err != nil { + return fmt.Errorf("applying: %w", err) + } + return appliedWithUploadMsg{Object: obj} + } +} + +type appliedMsg struct { + client.Object +} + +func applyCmd(ctx context.Context, res *client.Resource, obj client.Object) tea.Cmd { + return func() tea.Msg { + if err := res.Apply(obj, true); err != nil { + return fmt.Errorf("applying: %w", err) + } + return appliedMsg{Object: obj} + } +} + +type createdWithUploadMsg struct { + client.Object +} + +func createWithUploadCmd(ctx context.Context, res *client.Resource, obj client.Object, tarball *client.Tarball) tea.Cmd { + return func() tea.Msg { + if err := specifyUpload(obj, tarball); err != nil { + return fmt.Errorf("specifying upload: %w", err) + } + + lowerKind := strings.ToLower(obj.GetObjectKind().GroupVersionKind().Kind) + if obj.GetLabels() == nil { + obj.SetLabels(map[string]string{}) + } + obj.GetLabels()[lowerKind] = obj.GetName() + + list, err := res.List(obj.GetNamespace(), obj.GetObjectKind().GroupVersionKind().Version, &metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + lowerKind: obj.GetName(), + }).String(), + }) + if err != nil { + return fmt.Errorf("listing: %w", err) + } + + var version int + switch list := list.(type) { + case *apiv1.ModelList: + version, err = nextModelVersion(list) + if err != nil { + return fmt.Errorf("next model version: %w", err) + } + case *apiv1.DatasetList: + version, err = nextDatasetVersion(list) + if err != nil { + return fmt.Errorf("next dataset version: %w", err) + } + default: + return fmt.Errorf("unrecognized list type: %T", list) + } + + log.Printf("Next version: %v", version) + + obj.SetName(fmt.Sprintf("%v.v%v", obj.GetName(), version)) + obj.GetLabels()["version"] = fmt.Sprintf("%v", version) + if _, err := res.Create(obj.GetNamespace(), true, obj); err != nil { + return fmt.Errorf("creating: %w", err) + } + return createdWithUploadMsg{Object: obj} + } +} + +type objectReadyMsg struct { + client.Object +} + +type objectUpdateMsg struct { + client.Object +} + +func waitReadyCmd(ctx context.Context, res *client.Resource, obj client.Object) tea.Cmd { + return func() tea.Msg { + if err := res.WaitReady(ctx, obj, func(updatedObj client.Object) { + P.Send(objectUpdateMsg{Object: updatedObj}) + }); err != nil { + return fmt.Errorf("waiting to be ready: %w", err) + } + return objectReadyMsg{Object: obj} + } +} + +func nextModelVersion(list *apiv1.ModelList) (int, error) { + var highestVersion int + for _, item := range list.Items { + v, err := strconv.Atoi(item.GetLabels()["version"]) + if err != nil { + return 0, fmt.Errorf("version label to int: %w", err) + } + if v > highestVersion { + highestVersion = v + } + } + + return highestVersion + 1, nil +} + +func nextDatasetVersion(list *apiv1.DatasetList) (int, error) { + var highestVersion int + for _, item := range list.Items { + v, err := strconv.Atoi(item.GetLabels()["version"]) + if err != nil { + return 0, fmt.Errorf("version label to int: %w", err) + } + if v > highestVersion { + highestVersion = v + } + } + + return highestVersion + 1, nil +} + +type suspendedMsg struct { + error error +} + +func suspendCmd(ctx context.Context, res *client.Resource, obj client.Object) tea.Cmd { + return func() tea.Msg { + log.Println("Suspending") + _, err := res.Patch(obj.GetNamespace(), obj.GetName(), types.MergePatchType, []byte(`{"spec": {"suspend": true} }`), &metav1.PatchOptions{}) + if err != nil { + log.Printf("Error suspending: %v", err) + return suspendedMsg{error: err} + } + return suspendedMsg{} + } +} + +type deletedMsg struct { + name string + error error +} + +func deleteCmd(ctx context.Context, res *client.Resource, obj client.Object) tea.Cmd { + return func() tea.Msg { + name := obj.GetName() + + log.Println("Deleting") + _, err := res.Delete(obj.GetNamespace(), obj.GetName()) + if err != nil { + log.Printf("Error deleting: %v", err) + return deletedMsg{name: name, error: err} + } + + return deletedMsg{name: name} + } +} + +type readManifestMsg struct { + obj client.Object +} + +func readManifest(path string) tea.Cmd { + return func() tea.Msg { + log.Println("Reading manifest") + + manifest, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading file: %w", err) + } + + var obj client.Object + obj, err = client.Decode(manifest) + if err != nil { + return fmt.Errorf("decoding: %w", err) + } + + return readManifestMsg{ + obj: obj, + } + } +} diff --git a/internal/tui/delete.go b/internal/tui/delete.go new file mode 100644 index 00000000..a780c6ec --- /dev/null +++ b/internal/tui/delete.go @@ -0,0 +1,162 @@ +package tui + +import ( + "context" + "fmt" + "log" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/substratusai/substratus/internal/client" +) + +type DeleteModel struct { + // Cancellation + Ctx context.Context + + // Config + Scope string + Namespace Namespace + + // Clients + Client client.Interface + + resource *client.Resource + toDelete []client.Object + deleted []string + + Style lipgloss.Style + + // End times + goodbye string + finalError error +} + +func (m *DeleteModel) New() DeleteModel { + m.Style = appStyle + if m.toDelete == nil { + m.toDelete = []client.Object{} + } + return *m +} + +type deleteInitMsg struct{} + +func (m DeleteModel) Init() tea.Cmd { + //if len(m.Objects) == 0 { + // return listCmd(m.Ctx, m.Resource, m.Scope) + //} else { + // cmds := make([]tea.Cmd, 0, len(m.Objects)) + // for _, obj := range m.Objects { + // cmds = append(cmds, deleteCmd(m.Ctx, m.Resource, obj)) + // } + // return tea.Batch(cmds...) + //} + + return func() tea.Msg { return deleteInitMsg{} } +} + +func (m DeleteModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + + case deleteInitMsg: + log.Println("running init") + obj, err := scopeToObject(m.Scope) + if err != nil { + m.finalError = fmt.Errorf("scope to object: %w", err) + return m, nil + } + m.Namespace.Set(obj) + + res, err := m.Client.Resource(obj) + if err != nil { + m.finalError = fmt.Errorf("resource client: %w", err) + return m, nil + } + m.resource = res + + return m, getDeletionTargetsCmd(m.Ctx, m.resource, obj) + + case tea.KeyMsg: + log.Println("Received key msg:", msg.String()) + if msg.String() == "q" { + return m, tea.Quit + } + + case deletionListMsg: + var cmds []tea.Cmd + m.toDelete = msg.items + for _, obj := range msg.items { + log.Printf("to-delete: %v", obj.GetName()) + // TODO: Implement a confirmation flow. + cmds = append(cmds, deleteCmd(m.Ctx, m.resource, obj)) + } + return m, tea.Sequence(cmds...) + + case deletedMsg: + if msg.error != nil { + m.finalError = msg.error + } else { + m.deleted = append(m.deleted, msg.name) + } + if len(m.deleted) == len(m.toDelete) { + return m, tea.Quit + } + + case tea.WindowSizeMsg: + m.Style.Width(msg.Width) + } + + return m, nil +} + +// View returns a string based on data in the model. That string which will be +// rendered to the terminal. +func (m DeleteModel) View() (v string) { + defer func() { + v += helpStyle("Press \"q\" to quit") + v = m.Style.Render(v) + }() + + if m.finalError != nil { + v += errorStyle.Width(m.Style.GetWidth()-m.Style.GetHorizontalMargins()-10).Render("Error: "+m.finalError.Error()) + "\n" + return + } + + if m.goodbye != "" { + v += m.goodbye + "\n" + return + } + + for _, name := range m.deleted { + v += checkMark.String() + " " + name + ": deleted\n" + } + + return +} + +type deletionListMsg struct { + items []client.Object +} + +func getDeletionTargetsCmd(ctx context.Context, res *client.Resource, obj client.Object) tea.Cmd { + log.Printf("getting deletion targets: %v/%v", obj.GetNamespace(), obj.GetName()) + return func() tea.Msg { + if obj.GetName() != "" { + fetched, err := res.Get(obj.GetNamespace(), obj.GetName()) + if err != nil { + return fmt.Errorf("get: %w", err) + } + return deletionListMsg{items: []client.Object{fetched.(client.Object)}} + } else { + items, err := res.List(obj.GetNamespace(), "", &metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("list: %w", err) + } + log.Printf("%T", items) + panic("NOT IMPLEMENTED") + } + } +} diff --git a/internal/tui/get.go b/internal/tui/get.go new file mode 100644 index 00000000..de3f44bb --- /dev/null +++ b/internal/tui/get.go @@ -0,0 +1,369 @@ +package tui + +import ( + "context" + "fmt" + "log" + "math" + "slices" + "sort" + "strings" + + "github.com/charmbracelet/bubbles/spinner" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + + apiv1 "github.com/substratusai/substratus/api/v1" + "github.com/substratusai/substratus/internal/client" +) + +type GetModel struct { + // Cancellation + Ctx context.Context + + // Config + Scope string + Namespace string + + // Clients + Client client.Interface + + // End times + finalError error + + objects map[string]map[string]listedObject + + Style lipgloss.Style +} + +type listedObject struct { + object + spinner spinner.Model +} + +func newGetObjectMap() map[string]map[string]listedObject { + return map[string]map[string]listedObject{ + "notebooks": {}, + "datasets": {}, + "models": {}, + "servers": {}, + } +} + +func (m *GetModel) New() GetModel { + m.objects = newGetObjectMap() + m.Style = appStyle + return *m +} + +func (m GetModel) Init() tea.Cmd { + return watchCmd(m.Ctx, m.Client, m.Namespace, m.Scope) +} + +func (m GetModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.KeyMsg: + log.Println("Received key msg:", msg.String()) + if msg.String() == "q" { + return m, tea.Quit + } + + case watchMsg: + var cmd tea.Cmd + switch msg.Type { + case watch.Deleted: + delete(m.objects[msg.resource], msg.Object.(object).GetName()) + case watch.Error: + log.Printf("Watch error: %v", msg.Object) + default: + o := msg.Object.(client.Object) + name := o.GetName() + log.Printf("Watch event: %v: %v", msg.resource, name) + + lo := m.objects[msg.resource][name] + lo.object = msg.Object.(object) + if msg.Type == watch.Added { + lo.spinner = spinner.New(spinner.WithSpinner(spinner.MiniDot), spinner.WithStyle(activeSpinnerStyle)) + cmd = lo.spinner.Tick + } + m.objects[msg.resource][name] = lo + } + return m, cmd + + case spinner.TickMsg: + for kind := range m.objects { + for name := range m.objects[kind] { + o := m.objects[kind][name] + var cmd tea.Cmd + if o.spinner.ID() == msg.ID { + o.spinner, cmd = o.spinner.Update(msg) + m.objects[kind][name] = o + return m, cmd + } + } + } + return m, nil + + case tea.WindowSizeMsg: + m.Style.Width(msg.Width) + + case error: + m.finalError = msg + return m, nil + } + + return m, nil +} + +// View returns a string based on data in the model. That string which will be +// rendered to the terminal. +func (m GetModel) View() (v string) { + defer func() { + v = m.Style.Render(v) + }() + + if m.finalError != nil { + v += errorStyle.Render("Error: "+m.finalError.Error()) + "\n" + v += helpStyle("Press \"q\" to quit") + return v + } + + scopeResource, scopeName := splitScope(m.Scope) + + var total int + for _, resource := range []struct { + plural string + versioned bool + }{ + {plural: "notebooks", versioned: false}, + {plural: "datasets", versioned: true}, + {plural: "models", versioned: true}, + {plural: "servers", versioned: false}, + } { + if len(m.objects[resource.plural]) == 0 { + continue + } + + if scopeResource == "" { + v += resource.plural + "/" + "\n" + } + + var names []string + for name := range m.objects[resource.plural] { + names = append(names, name) + total++ + } + sort.Strings(names) + + if !resource.versioned { + for _, name := range names { + o := m.objects[resource.plural][name] + + var indicator string + if o.GetStatusReady() { + indicator = checkMark.String() + } else { + indicator = o.spinner.View() + } + v += "" + indicator + " " + name + "\n" + } + } else { + type objectVersions struct { + unversionedName string + versions []listedObject + } + + var groups []objectVersions + + var lastUnversionedName string + // var longestName int + const longestName = 30 + for _, name := range names { + o := m.objects[resource.plural][name] + lowerKind := strings.TrimSuffix(resource.plural, "s") + unversionedName := o.GetLabels()[lowerKind] + + if unversionedName != lastUnversionedName { + groups = append(groups, objectVersions{ + unversionedName: unversionedName, + versions: []listedObject{o}, + }) + } else { + groups[len(groups)-1].versions = append(groups[len(groups)-1].versions, o) + } + + lastUnversionedName = unversionedName + //if n := len(name); n > longestName { + // longestName = n + 6 + //} + } + + for gi, g := range groups { + type versionDisplay struct { + indicator string + version string + } + var displayVersions []versionDisplay + for _, o := range g.versions { + version := o.GetLabels()["version"] + + var indicator string + if o.GetStatusReady() { + indicator = checkMark.String() + } else if c := meta.FindStatusCondition(*o.GetConditions(), apiv1.ConditionComplete); c != nil && c.Reason == apiv1.ReasonJobFailed { + indicator = xMark.String() + } else { + indicator = o.spinner.View() + } + displayVersions = append(displayVersions, versionDisplay{ + indicator: indicator, + version: version, + }) + } + + // Latest first + slices.Reverse(displayVersions) + + var otherVersions []string + for _, other := range displayVersions[1:] { + otherVersions = append(otherVersions, fmt.Sprintf("%v.v%v", other.indicator, other.version)) + } + + primary := displayVersions[0].indicator + " " + + g.unversionedName + ".v" + + displayVersions[0].version + + verWidth := int(math.Min(float64(60), float64(m.Style.GetWidth()-m.Style.GetHorizontalMargins()-longestName-18))) + v += lipgloss.JoinHorizontal( + lipgloss.Top, + lipgloss.NewStyle().Width(longestName).MarginLeft(0).MarginRight(2).Align(lipgloss.Left).Render(primary), + lipgloss.NewStyle().Width(verWidth).MarginRight(4).Align(lipgloss.Right).Render(strings.Join(otherVersions, " ")), + ) + if gi < len(groups) { + v += "\n" + } + } + + } + v += "\n" + } + + if scopeName == "" { + v += fmt.Sprintf("\nTotal: %v\n", total) + } + + v += helpStyle("Press \"q\" to quit") + + return v +} + +type watchMsg struct { + watch.Event + resource string +} + +type object interface { + client.Object + GetConditions() *[]metav1.Condition + GetStatusReady() bool +} + +func watchCmd(ctx context.Context, c client.Interface, namespace, scope string) tea.Cmd { + pluralName := func(s string) string { + return strings.ToLower(s) + "s" + } + + return func() tea.Msg { + log.Println("Starting watch") + + objs, err := scopeToObjects(scope) + if err != nil { + return fmt.Errorf("parsing search term: %v", err) + } + + for _, obj := range objs { + res, err := c.Resource(obj) + if err != nil { + return fmt.Errorf("resource client: %w", err) + } + + kind := obj.GetObjectKind().GroupVersionKind().Kind + log.Printf("Starting watch: %v", kind) + + w, err := res.Watch(ctx, namespace, obj, &metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("watch: %w", err) + } + go func() { + for event := range w.ResultChan() { + P.Send(watchMsg{Event: event, resource: pluralName(kind)}) + } + }() + } + + return nil + } +} + +// scopeToObjects maps a scope string to a slice of objects. +// "" --> All Substratus kinds +// "models" --> All Models +// "models/" --> Single Model +func scopeToObjects(scope string) ([]client.Object, error) { + if scope == "" { + return []client.Object{ + &apiv1.Notebook{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Notebook"}}, + &apiv1.Dataset{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Dataset"}}, + &apiv1.Model{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Model"}}, + &apiv1.Server{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Server"}}, + }, nil + } + + singleObj, err := scopeToObject(scope) + if err != nil { + return nil, err + } + + return []client.Object{singleObj}, nil +} + +func scopeToObject(scope string) (client.Object, error) { + res, name := splitScope(scope) + if res == "" && name == "" { + return nil, fmt.Errorf("Invalid scope: %v", scope) + } + + var obj client.Object + switch res { + case "notebooks": + obj = &apiv1.Notebook{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Notebook"}} + case "datasets": + obj = &apiv1.Dataset{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Dataset"}} + case "models": + obj = &apiv1.Model{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Model"}} + case "servers": + obj = &apiv1.Server{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Server"}} + default: + return nil, fmt.Errorf("Invalid scope: %v", scope) + } + + if name != "" { + obj.SetName(name) + } + + return obj, nil +} + +func splitScope(scope string) (string, string) { + split := strings.Split(scope, "/") + if len(split) == 1 { + return split[0], "" + } + if len(split) == 2 { + return split[0], split[1] + } + return "", "" +} diff --git a/internal/tui/infer_chat.go b/internal/tui/infer_chat.go new file mode 100644 index 00000000..42d3781e --- /dev/null +++ b/internal/tui/infer_chat.go @@ -0,0 +1,92 @@ +package tui + +import ( + "fmt" + "strings" + + "github.com/charmbracelet/bubbles/textarea" + "github.com/charmbracelet/bubbles/viewport" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" +) + +type ( + errMsg error +) + +type ChatModel struct { + viewport viewport.Model + messages []string + textarea textarea.Model + senderStyle lipgloss.Style + err error +} + +func (m ChatModel) Init() tea.Cmd { + ta := textarea.New() + ta.Placeholder = "Send a message..." + ta.Focus() + + ta.Prompt = "┃ " + ta.CharLimit = 280 + + ta.SetWidth(30) + ta.SetHeight(3) + + // Remove cursor line styling + ta.FocusedStyle.CursorLine = lipgloss.NewStyle() + + ta.ShowLineNumbers = false + + vp := viewport.New(30, 5) + vp.SetContent(`Welcome to the chat room! +Type a message and press Enter to send.`) + + ta.KeyMap.InsertNewline.SetEnabled(false) + + m.textarea = ta + m.messages = []string{} + m.viewport = vp + m.senderStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("5")) + + return textarea.Blink +} + +func (m ChatModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + var ( + tiCmd tea.Cmd + vpCmd tea.Cmd + ) + + m.textarea, tiCmd = m.textarea.Update(msg) + m.viewport, vpCmd = m.viewport.Update(msg) + + switch msg := msg.(type) { + case tea.KeyMsg: + switch msg.Type { + case tea.KeyCtrlC, tea.KeyEsc: + fmt.Println(m.textarea.Value()) + return m, tea.Quit + case tea.KeyEnter: + m.messages = append(m.messages, m.senderStyle.Render("You: ")+m.textarea.Value()) + m.viewport.SetContent(strings.Join(m.messages, "\n")) + m.textarea.Reset() + m.viewport.GotoBottom() + } + + // We handle errors just like any other message + case errMsg: + m.err = msg + return m, nil + } + + return m, tea.Batch(tiCmd, vpCmd) +} + +func (m ChatModel) View() string { + return fmt.Sprintf( + "%s\n\n%s", + m.viewport.View(), + m.textarea.View(), + ) + "\n\n" +} diff --git a/internal/tui/manifests.go b/internal/tui/manifests.go new file mode 100644 index 00000000..c7097aa8 --- /dev/null +++ b/internal/tui/manifests.go @@ -0,0 +1,187 @@ +package tui + +import ( + "bytes" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + + apiv1 "github.com/substratusai/substratus/api/v1" + "github.com/substratusai/substratus/internal/client" +) + +type manifestsModel struct { + Path string + Filename string + + // Kinds is a list of manifest kinds to include in results, + // ordered by preference. + Kinds []string + + reading status + + Style lipgloss.Style +} + +// New initializes all internal fields. +func (m *manifestsModel) New() manifestsModel { + return *m +} + +func (m manifestsModel) Active() bool { + return m.reading == inProgress +} + +func (m manifestsModel) Init() tea.Cmd { + return tea.Sequence( + func() tea.Msg { return manifestsInitMsg{} }, + findSubstratusManifests(m.Path, m.Filename), + ) +} + +type manifestsInitMsg struct{} + +func (m manifestsModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case manifestsInitMsg: + log.Println("Initializing manifests") + m.reading = inProgress + + case readManifestMsg: + m.reading = completed + + return m, func() tea.Msg { return manifestSelectedMsg{obj: msg.obj} } + + case substratusManifestsMsg: + m.reading = completed + + var n int + var single client.Object + for _, k := range m.Kinds { + items := msg.manifests[k] + if single == nil && len(items) > 0 { + single = items[0] + } + n += len(items) + } + + log.Printf("Found (filtered) manifests: %v", n) + + if n == 0 { + return m, func() tea.Msg { return fmt.Errorf("No substratus Server kinds found in *.yaml") } + } else if n == 1 { + return m, func() tea.Msg { return manifestSelectedMsg{obj: single} } + } else { + // TODO: Selector + return m, func() tea.Msg { return manifestSelectedMsg{obj: single} } + } + + } + + return m, nil +} + +// View returns a string based on data in the model. That string which will be +// rendered to the terminal. +func (m manifestsModel) View() (v string) { + defer func() { + if v != "" { + v = m.Style.Render(v) + } + }() + + if m.reading == inProgress { + v += "Reading manifests..." + } + + return +} + +type manifestSelectedMsg struct { + obj client.Object +} + +type substratusManifestsMsg struct { + manifests map[string][]client.Object +} + +func findSubstratusManifests(path, filename string) tea.Cmd { + return func() tea.Msg { + msg := substratusManifestsMsg{ + manifests: map[string][]client.Object{}, + } + + var fp string + if filename != "" { + fp = filepath.Join(path, filename) + manifest, err := os.ReadFile(fp) + if err != nil { + return fmt.Errorf("reading file: %w", err) + } + if err := manifestToObjects(manifest, msg.manifests); err != nil { + return fmt.Errorf("reading manifests in file: %v: %w", filename, err) + } + } else { + if path == "" { + var err error + path, err = os.Getwd() + if err != nil { + return err + } + } + + fp = filepath.Join(path, "*.yaml") + matches, err := filepath.Glob(fp) + if err != nil { + return err + } + for _, p := range matches { + manifest, err := os.ReadFile(p) + if err != nil { + return fmt.Errorf("reading file: %w", err) + } + + if err := manifestToObjects(manifest, msg.manifests); err != nil { + return fmt.Errorf("reading manifests in file: %v: %w", p, err) + } + } + } + if len(msg.manifests) == 0 { + return fmt.Errorf("No manifests found: %v", fp) + } + return msg + } +} + +func manifestToObjects(manifest []byte, m map[string][]client.Object) error { + split := bytes.Split(manifest, []byte("---\n")) + for _, doc := range split { + if strings.TrimSpace(string(doc)) == "" { + continue + } + + obj, err := client.Decode(doc) + if err != nil { + return fmt.Errorf("decoding: %w", err) + } + if obj == nil { + return nil + } + + switch t := obj.(type) { + case *apiv1.Model, *apiv1.Dataset, *apiv1.Server, *apiv1.Notebook: + kind := t.GetObjectKind().GroupVersionKind().Kind + if m[kind] == nil { + m[kind] = make([]client.Object, 0) + } + m[kind] = append(m[kind], obj) + } + } + + return nil +} diff --git a/internal/tui/notebook.go b/internal/tui/notebook.go new file mode 100644 index 00000000..ea681881 --- /dev/null +++ b/internal/tui/notebook.go @@ -0,0 +1,330 @@ +package tui + +import ( + "context" + "errors" + "fmt" + "log" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/pkg/browser" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + + apiv1 "github.com/substratusai/substratus/api/v1" + "github.com/substratusai/substratus/internal/client" +) + +type NotebookModel struct { + // Cancellation + Ctx context.Context + + // Config + Path string + Filename string + Namespace Namespace + NoOpenBrowser bool + + // Clients + Client client.Interface + K8s *kubernetes.Clientset + + // Current notebook + notebook *apiv1.Notebook + resource *client.Resource + + // Proceses + manifests manifestsModel + upload uploadModel + readiness readinessModel + pods podsModel + + // File syncing + syncingFiles status + currentSyncingFile string + lastSyncFailure error + + // Ready to open browser + portForwarding status + localURL string + + // End times + quitting bool + goodbye string + finalError error + + Style lipgloss.Style +} + +func (m NotebookModel) cleanupAndQuitCmd() tea.Msg { + m.upload.cleanup() + return tea.Quit() +} + +func (m *NotebookModel) New() NotebookModel { + m.manifests = (&manifestsModel{ + Path: m.Path, + Filename: m.Filename, + Kinds: []string{"Notebook", "Model", "Dataset"}, + }).New() + m.upload = (&uploadModel{ + Ctx: m.Ctx, + Client: m.Client, + Path: m.Path, + Mode: uploadModeApply, + }).New() + m.readiness = (&readinessModel{ + Ctx: m.Ctx, + Client: m.Client, + }).New() + m.pods = (&podsModel{ + Ctx: m.Ctx, + Client: m.Client, + K8s: m.K8s, + }).New() + + m.Style = appStyle + + return *m +} + +func (m NotebookModel) Init() tea.Cmd { + // return readManifest(filepath.Join(m.Path, m.Filename)) + return m.manifests.Init() +} + +func (m NotebookModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + var cmds []tea.Cmd + + log.Printf("MSG: %T", msg) + + { + mdl, cmd := m.manifests.Update(msg) + m.manifests = mdl.(manifestsModel) + cmds = append(cmds, cmd) + } + + { + mdl, cmd := m.upload.Update(msg) + m.upload = mdl.(uploadModel) + cmds = append(cmds, cmd) + } + + { + mdl, cmd := m.readiness.Update(msg) + m.readiness = mdl.(readinessModel) + cmds = append(cmds, cmd) + } + + { + mdl, cmd := m.pods.Update(msg) + m.pods = mdl.(podsModel) + cmds = append(cmds, cmd) + } + + switch msg := msg.(type) { + case manifestSelectedMsg: + m.Namespace.Set(msg.obj) + nb, err := client.NotebookForObject(msg.obj) + if err != nil { + m.finalError = fmt.Errorf("determining notebook: %w", err) + } + nb.Spec.Suspend = ptr.To(false) + m.notebook = nb + + res, err := m.Client.Resource(m.notebook) + if err != nil { + m.finalError = fmt.Errorf("resource client: %w", err) + } + m.resource = res + + m.upload.Object = m.notebook + m.upload.Resource = m.resource + cmds = append(cmds, m.upload.Init()) + + case tea.KeyMsg: + log.Println("Received key msg:", msg.String()) + if m.quitting { + switch msg.String() { + case "esc": + if m.finalError == nil { + m.quitting = false + } + + // "Leave be" results in issues where a build will eventually replace the Notebook + // and the command will error out due to a failure on the previous notebook nbwatch + // command... revisit later. + // + // case "l": + // cmds = append(cmds, m.cleanupAndQuitCmd) + case "s": + cmds = append(cmds, suspendCmd(context.Background(), m.resource, m.notebook)) + case "d": + cmds = append(cmds, deleteCmd(context.Background(), m.resource, m.notebook)) + } + } else { + if msg.String() == "q" { + m.quitting = true + } + } + + case suspendedMsg: + if msg.error != nil { + m.finalError = msg.error + } else { + m.goodbye = "Notebook suspended." + } + cmds = append(cmds, m.cleanupAndQuitCmd) + + case deletedMsg: + if msg.error != nil { + m.finalError = msg.error + } else { + m.goodbye = "Notebook deleted." + } + cmds = append(cmds, m.cleanupAndQuitCmd) + + case tarballUploadedMsg: + m.notebook = msg.Object.(*apiv1.Notebook) + + m.readiness.Object = m.notebook + m.readiness.Resource = m.resource + m.pods.Object = m.notebook + m.pods.Resource = m.resource + cmds = append(cmds, + m.readiness.Init(), + m.pods.Init(), + ) + + case objectReadyMsg: + m.notebook = msg.Object.(*apiv1.Notebook) + m.syncingFiles = inProgress + cmds = append(cmds, + notebookSyncFilesCmd(m.Ctx, m.Client, m.notebook.DeepCopy(), m.Path), + portForwardCmd(m.Ctx, m.Client, client.PodForNotebook(m.notebook), client.ForwardedPorts{Local: 8888, Pod: 8888}), + ) + + case notebookFileSyncMsg: + if msg.complete { + m.currentSyncingFile = "" + } else { + m.currentSyncingFile = msg.file + } + if msg.error != nil { + m.lastSyncFailure = msg.error + } else { + m.lastSyncFailure = nil + } + + case portForwardReadyMsg: + cmds = append(cmds, notebookOpenInBrowser(m.notebook.DeepCopy())) + + case localURLMsg: + m.localURL = string(msg) + + case tea.WindowSizeMsg: + m.Style.Width(msg.Width) + innerWidth := m.Style.GetWidth() - m.Style.GetHorizontalPadding() + m.upload.Style = lipgloss.NewStyle().Width(innerWidth) + m.readiness.Style = lipgloss.NewStyle().Width(innerWidth) + m.pods.SetStyle(logStyle.Width(innerWidth)) + + case error: + log.Printf("Error message: %v", msg) + m.finalError = msg + m.quitting = true + } + + return m, tea.Batch(cmds...) +} + +// View returns a string based on data in the model. That string which will be +// rendered to the terminal. +func (m NotebookModel) View() (v string) { + defer func() { + v = m.Style.Render(v) + }() + + if m.finalError != nil { + v += errorStyle.Width(m.Style.GetWidth()-m.Style.GetHorizontalMargins()-10).Render("Error: "+m.finalError.Error()) + "\n" + v += helpStyle("Press \"s\" to suspend, \"d\" to delete") + // v += helpStyle("Press \"l\" to leave be, \"s\" to suspend, \"d\" to delete") + return v + } + + if m.goodbye != "" { + v += m.goodbye + "\n" + return v + } + + if m.quitting { + v += "Quitting...\n" + v += helpStyle("Press \"s\" to suspend, \"d\" to delete, \"ESC\" to cancel") + // v += helpStyle("Press \"l\" to leave be, \"s\" to suspend, \"d\" to delete, \"ESC\" to cancel") + return v + } + + v += m.manifests.View() + v += m.upload.View() + v += m.readiness.View() + v += m.pods.View() + + if m.syncingFiles == inProgress { + v += "\n" + if m.currentSyncingFile != "" { + v += fmt.Sprintf("Syncing from notebook: %v\n", m.currentSyncingFile) + } else { + v += "Watching for files to sync...\n" + } + if m.lastSyncFailure != nil { + v += errorStyle.Render("Sync failed: "+m.lastSyncFailure.Error()) + "\n\n" + } + } + + if m.portForwarding == inProgress { + v += "Port-forwarding...\n" + } + + if m.localURL != "" && m.portForwarding == inProgress { + v += "\n" + v += fmt.Sprintf("Notebook URL: %v\n", m.localURL) + } + + v += helpStyle("Press \"q\" to quit") + + return v +} + +type notebookFileSyncMsg struct { + file string + complete bool + error error +} + +func notebookSyncFilesCmd(ctx context.Context, c client.Interface, nb *apiv1.Notebook, dir string) tea.Cmd { + return func() tea.Msg { + if err := c.SyncFilesFromNotebook(ctx, nb, dir, LogFile, func(file string, complete bool, syncErr error) { + P.Send(notebookFileSyncMsg{ + file: file, + complete: complete, + error: syncErr, + }) + }); err != nil { + if !errors.Is(err, context.Canceled) { + return err + } + } + return nil + } +} + +func notebookOpenInBrowser(nb *apiv1.Notebook) tea.Cmd { + return func() tea.Msg { + // TODO(nstogner): Grab token from Notebook status. + url := "http://localhost:8888?token=default" + log.Printf("Opening browser to %s\n", url) + browser.OpenURL(url) + return localURLMsg(url) + } +} diff --git a/internal/tui/pods.go b/internal/tui/pods.go new file mode 100644 index 00000000..e70eaf21 --- /dev/null +++ b/internal/tui/pods.go @@ -0,0 +1,246 @@ +package tui + +import ( + "bufio" + "context" + "fmt" + "log" + "sort" + "strings" + + "github.com/charmbracelet/bubbles/viewport" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + + "github.com/substratusai/substratus/internal/client" +) + +type podsModel struct { + // Cancellation + Ctx context.Context + + // Clients + Client client.Interface + Resource *client.Resource + K8s *kubernetes.Clientset + + Object client.Object + + watchingPods status + + // Watch Pods + // map[role][podName] + pods map[string]map[string]podInfo + + // End times + finalError error + + Style lipgloss.Style +} + +type podInfo struct { + lastEvent watch.EventType + pod *corev1.Pod + + logs string + logsStarted bool + logsViewport viewport.Model +} + +// New initializes all internal fields. +func (m *podsModel) New() podsModel { + m.pods = map[string]map[string]podInfo{} + return *m +} + +func (m podsModel) Active() bool { + return m.watchingPods == inProgress +} + +func (m podsModel) Init() tea.Cmd { + return tea.Sequence( + func() tea.Msg { return podsInitMsg{} }, + watchPods(m.Ctx, m.Client, m.Object.DeepCopyObject().(client.Object)), + ) +} + +type podsInitMsg struct{} + +func (m podsModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case podsInitMsg: + log.Println("Initializing pods") + m.watchingPods = inProgress + + case podWatchMsg: + role := msg.Pod.Labels["role"] + + if _, ok := m.pods[role][msg.Pod.Name]; !ok { + m.pods[role] = map[string]podInfo{} + } + pi := m.pods[role][msg.Pod.Name] + pi.lastEvent = msg.Type + pi.pod = msg.Pod.DeepCopy() + + containerName := pi.pod.Annotations["kubectl.kubernetes.io/default-container"] + + var cmd tea.Cmd + if !pi.logsStarted { + for _, status := range pi.pod.Status.ContainerStatuses { + if status.Name == containerName && status.Ready { + log.Printf("Getting logs for Pod container: %v", status.Name) + cmd = getLogs(m.Ctx, m.K8s, pi.pod, containerName) + pi.logsStarted = true + pi.logsViewport = viewport.New(m.Style.GetWidth()-10, 7) + pi.logsViewport.Style = m.Style + break + } else { + log.Printf("Skipping logs for container: %v (Ready = %v)", status.Name, status.Ready) + } + } + } + + m.pods[msg.Pod.Labels["role"]][msg.Pod.Name] = pi + return m, cmd + + case podLogsMsg: + pi := m.pods[msg.role][msg.name] + // Fix the rendering of line-rewrites by always appending lines. + logs := msg.logs + logs = strings.ReplaceAll(logs, "\r", "\n") + logs = strings.TrimRight(logs, "\n") + logs = logs + "\n" + pi.logs += logs + pi.logsViewport.SetContent(lipgloss.NewStyle().Width(m.Style.GetWidth() - m.Style.GetHorizontalPadding()).Render(pi.logs) /*wordwrap.String(pi.logs, m.width-14)*/) + pi.logsViewport.GotoBottom() + m.pods[msg.role][msg.name] = pi + return m, nil + + case error: + m.finalError = msg + return m, nil + } + + return m, nil +} + +// View returns a string based on data in the model. That string which will be +// rendered to the terminal. +func (m podsModel) View() (v string) { + if m.watchingPods == inProgress { + v += "Pods:\n" + + roles := []string{"build", "run"} + + for _, role := range roles { + var pods []podInfo + for _, p := range m.pods[role] { + pods = append(pods, p) + } + sort.Slice(pods, func(i, j int) bool { + return pods[i].pod.CreationTimestamp.Before(&pods[j].pod.CreationTimestamp) + }) + for _, p := range pods { + if p.lastEvent == watch.Deleted { + continue + } + v += "> " + strings.Title(p.pod.Labels["role"]) + " (" + string(p.pod.Status.Phase) + ")\n" + if p.pod.Status.Phase != corev1.PodSucceeded { + v += "\n" + p.logsViewport.View() + "\n" + } + } + } + } + + return v +} + +func (m *podsModel) SetStyle(s lipgloss.Style) { + m.Style = s + for role := range m.pods { + for name := range m.pods[role] { + pi := m.pods[role][name] + if pi.logsViewport.Width > 0 { + pi.logsViewport.Width = s.GetWidth() + pi.logsViewport.SetContent(lipgloss.NewStyle().Width(s.GetWidth() - s.GetHorizontalMargins()).Render(pi.logs)) + pi.logsViewport.Style = s + m.pods[role][name] = pi + } + } + } +} + +type podWatchMsg struct { + Type watch.EventType + Pod *corev1.Pod +} + +func watchPods(ctx context.Context, c client.Interface, obj client.Object) tea.Cmd { + return func() tea.Msg { + log.Println("Starting Pod watch") + + pods, err := c.Resource(&corev1.Pod{TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}}) + if err != nil { + return fmt.Errorf("pods client: %w", err) + } + + w, err := pods.Watch(ctx, obj.GetNamespace(), nil, &metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + strings.ToLower(obj.GetObjectKind().GroupVersionKind().Kind): obj.GetName(), + //"role": role, + }).String(), + }) + if err != nil { + return fmt.Errorf("watch: %w", err) + } + go func() { + for event := range w.ResultChan() { + switch event.Type { + case watch.Added, watch.Modified, watch.Deleted: + pod := event.Object.(*corev1.Pod) + log.Printf("Pod event: %s: %s", pod.Name, event.Type) + P.Send(podWatchMsg{Type: event.Type, Pod: pod}) + } + } + }() + + return nil + } +} + +type podLogsMsg struct { + role string + name string + logs string +} + +func getLogs(ctx context.Context, k8s *kubernetes.Clientset, pod *corev1.Pod, container string) tea.Cmd { + return func() tea.Msg { + log.Printf("Starting to get logs for pod: %v", pod.Name) + req := k8s.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{ + Container: container, + Follow: true, + Timestamps: false, + }) + logs, err := req.Stream(ctx) + if err != nil { + return err + } + + scanner := bufio.NewScanner(logs) + for scanner.Scan() { + logs := scanner.Text() + log.Printf("Pod logs for: %v: %q", pod.Name, logs) + P.Send(podLogsMsg{role: pod.Labels["role"], name: pod.Name, logs: logs}) + } + if err := scanner.Err(); err != nil { + return err + } + return nil + } +} diff --git a/internal/tui/portforward.go b/internal/tui/portforward.go new file mode 100644 index 00000000..f8eb503f --- /dev/null +++ b/internal/tui/portforward.go @@ -0,0 +1,63 @@ +package tui + +import ( + "context" + "log" + "math" + "time" + + tea "github.com/charmbracelet/bubbletea" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/runtime" + + "github.com/substratusai/substratus/internal/client" +) + +type portForwardReadyMsg struct{} + +func portForwardCmd(ctx context.Context, c client.Interface, podRef types.NamespacedName, ports client.ForwardedPorts) tea.Cmd { + return func() tea.Msg { + const maxRetries = 3 + for i := 0; i < maxRetries; i++ { + portFwdCtx, cancelPortFwd := context.WithCancel(ctx) + defer cancelPortFwd() // Avoid a context leak + runtime.ErrorHandlers = []func(err error){ + func(err error) { + // Cancel a broken port forward to attempt to restart the port-forward. + log.Printf("Port-forward error: %v", err) + cancelPortFwd() + }, + } + + // portForward will close the ready channel when it returns. + // so we only use the outer ready channel once. On restart of the portForward, + // we use a new channel. + ready := make(chan struct{}) + go func() { + log.Println("Waiting for port-forward to be ready") + <-ready + log.Println("Port-forward ready") + P.Send(portForwardReadyMsg{}) + }() + + if err := c.PortForward(portFwdCtx, LogFile, podRef, ports, ready); err != nil { + log.Printf("Port-forward returned an error: %v", err) + } + + // Check if the command's context is cancelled, if so, + // avoid restarting the port forward. + if err := ctx.Err(); err != nil { + log.Printf("Context done, not attempting to restart port-forward: %v", err.Error()) + return nil + } + + cancelPortFwd() // Avoid a build up of contexts before returning. + backoff := time.Duration(math.Pow(2, float64(i))) * time.Second + log.Printf("Restarting port forward (index = %v), after backoff: %s", i, backoff) + time.Sleep(backoff) + } + log.Println("Done trying to port-forward") + + return nil + } +} diff --git a/internal/tui/readiness.go b/internal/tui/readiness.go new file mode 100644 index 00000000..70db4b6b --- /dev/null +++ b/internal/tui/readiness.go @@ -0,0 +1,100 @@ +package tui + +import ( + "context" + "fmt" + "log" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/substratusai/substratus/internal/client" +) + +type readinessModel struct { + // Cancellation + Ctx context.Context + + // Clients + Client client.Interface + Resource *client.Resource + + Object client.Object + + // Readiness + waiting status + + Style lipgloss.Style +} + +// New initializes all internal fields. +func (m *readinessModel) New() readinessModel { + m.Style = lipgloss.NewStyle() + return *m +} + +func (m readinessModel) Active() bool { + return m.waiting == inProgress +} + +func (m readinessModel) Init() tea.Cmd { + return tea.Sequence( + func() tea.Msg { return readinessInitMsg{} }, + waitReadyCmd(m.Ctx, m.Resource, m.Object.DeepCopyObject().(client.Object)), + ) +} + +type readinessInitMsg struct{} + +func (m readinessModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case readinessInitMsg: + log.Println("Initializing readiness") + m.waiting = inProgress + + case objectUpdateMsg: + m.Object = msg.Object + + case objectReadyMsg: + m.waiting = completed + m.Object = msg.Object + } + + return m, nil +} + +// View returns a string based on data in the model. That string which will be +// rendered to the terminal. +func (m readinessModel) View() (v string) { + defer func() { + if v != "" { + v = m.Style.Render(v) + } + }() + + if m.waiting == inProgress { + kind := m.Object.GetObjectKind().GroupVersionKind().Kind + v += fmt.Sprintf("%v:\n", kind) + + if w, ok := m.Object.(interface { + GetConditions() *[]metav1.Condition + }); ok { + for _, c := range *w.GetConditions() { + var prefix, suffix string + if c.Status == metav1.ConditionTrue { + prefix = checkMark.String() + " " + } else { + prefix = xMark.String() + " " + suffix = " (" + c.Reason + ")" + } + v += lipgloss.NewStyle().Width(m.Style.GetWidth() - m.Style.GetHorizontalPadding()). + Render(prefix + c.Type + suffix) + v += "\n" + } + } + + } + + return v +} diff --git a/internal/tui/run.go b/internal/tui/run.go new file mode 100644 index 00000000..fb7092a0 --- /dev/null +++ b/internal/tui/run.go @@ -0,0 +1,176 @@ +package tui + +import ( + "context" + "encoding/json" + "fmt" + "log" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "k8s.io/client-go/kubernetes" + + "github.com/substratusai/substratus/internal/client" +) + +type RunModel struct { + // Cancellation + Ctx context.Context + + // Clients + K8s *kubernetes.Clientset + Client client.Interface + + // Config + Path string + Filename string + Namespace Namespace + + // Focal object + object client.Object + resource *client.Resource + + // Processes + manifests manifestsModel + upload uploadModel + readiness readinessModel + pods podsModel + + // End times + finalError error + + Style lipgloss.Style +} + +func (m *RunModel) New() RunModel { + m.manifests = (&manifestsModel{ + Path: m.Path, + Filename: m.Filename, + Kinds: []string{"Model", "Dataset"}, + }).New() + m.upload = (&uploadModel{ + Ctx: m.Ctx, + Client: m.Client, + Path: m.Path, + Mode: uploadModeCreate, + }).New() + m.readiness = (&readinessModel{ + Ctx: m.Ctx, + Client: m.Client, + }).New() + m.pods = (&podsModel{ + Ctx: m.Ctx, + Client: m.Client, + K8s: m.K8s, + }).New() + m.Style = appStyle + return *m +} + +func (m RunModel) Init() tea.Cmd { + return m.manifests.Init() +} + +func (m RunModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + var cmds []tea.Cmd + + log.Printf("MSG: %T", msg) + { + mdl, cmd := m.manifests.Update(msg) + m.manifests = mdl.(manifestsModel) + cmds = append(cmds, cmd) + } + + { + mdl, cmd := m.upload.Update(msg) + m.upload = mdl.(uploadModel) + cmds = append(cmds, cmd) + } + + { + mdl, cmd := m.readiness.Update(msg) + m.readiness = mdl.(readinessModel) + cmds = append(cmds, cmd) + } + + { + mdl, cmd := m.pods.Update(msg) + m.pods = mdl.(podsModel) + cmds = append(cmds, cmd) + } + + switch msg := msg.(type) { + case manifestSelectedMsg: + m.object = msg.obj + m.Namespace.Set(m.object) + + res, err := m.Client.Resource(m.object) + if err != nil { + b, _ := json.Marshal(m.object) + log.Println("............", string(b)) + + m.finalError = fmt.Errorf("resource client: %w", err) + break + } + m.resource = res + + m.upload.Object = m.object + m.upload.Resource = m.resource + cmds = append(cmds, m.upload.Init()) + + case tea.KeyMsg: + log.Println("Received key msg:", msg.String()) + if msg.String() == "q" { + cmds = append(cmds, tea.Quit) + } + + case tarballUploadedMsg: + m.object = msg.Object + + m.readiness.Object = m.object + m.readiness.Resource = m.resource + m.pods.Object = m.object + m.pods.Resource = m.resource + cmds = append(cmds, + m.readiness.Init(), + m.pods.Init(), + ) + + case tea.WindowSizeMsg: + m.Style.Width(msg.Width) + innerWidth := m.Style.GetWidth() - m.Style.GetHorizontalPadding() + // NOTE: Use background coloring for style debugging. + m.upload.Style = lipgloss.NewStyle().Width(innerWidth) //.Background(lipgloss.Color("12")) + m.readiness.Style = lipgloss.NewStyle().Width(innerWidth) //.Background(lipgloss.Color("202")) + m.pods.SetStyle(logStyle.Copy().Width(innerWidth)) //.Background(lipgloss.Color("86"))) + + case error: + log.Printf("Error message: %v", msg) + m.finalError = msg + } + + return m, tea.Batch(cmds...) +} + +// View returns a string based on data in the model. That string which will be +// rendered to the terminal. +func (m RunModel) View() (v string) { + defer func() { + v = m.Style.Render(v) + }() + + if m.finalError != nil { + v += errorStyle.Width(m.Style.GetWidth()-m.Style.GetHorizontalPadding()).Render("Error: "+m.finalError.Error()) + "\n" + v += helpStyle("Press \"q\" to quit") + return v + } + + v += m.manifests.View() + v += m.upload.View() + v += m.readiness.View() + v += m.pods.View() + + v += helpStyle("Press \"q\" to quit") + + return v +} diff --git a/internal/tui/serve.go b/internal/tui/serve.go new file mode 100644 index 00000000..dfe390e5 --- /dev/null +++ b/internal/tui/serve.go @@ -0,0 +1,281 @@ +package tui + +import ( + "context" + "fmt" + "log" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/pkg/browser" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + + apiv1 "github.com/substratusai/substratus/api/v1" + "github.com/substratusai/substratus/internal/client" +) + +type ServeModel struct { + // Cancellation + Ctx context.Context + + // Config + Namespace Namespace + Filename string + NoOpenBrowser bool + + // Clients + Client client.Interface + K8s *kubernetes.Clientset + + // Current Server + server *apiv1.Server + resource *client.Resource + readyPod *corev1.Pod + + applying status + + manifests manifestsModel + readiness readinessModel + pods podsModel + + // Ready to open browser + portForwarding status + localURL string + + Style lipgloss.Style + + // End times + quitting bool + goodbye string + finalError error +} + +func (m *ServeModel) New() ServeModel { + m.manifests = (&manifestsModel{ + Kinds: []string{"Server"}, + }).New() + m.readiness = (&readinessModel{ + Ctx: m.Ctx, + Client: m.Client, + }).New() + m.pods = (&podsModel{ + Ctx: m.Ctx, + Client: m.Client, + K8s: m.K8s, + }).New() + + m.Style = appStyle + + return *m +} + +func (m ServeModel) Init() tea.Cmd { + return m.manifests.Init() +} + +func (m ServeModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + var cmds []tea.Cmd + + log.Printf("MSG: %T", msg) + + { + mdl, cmd := m.manifests.Update(msg) + m.manifests = mdl.(manifestsModel) + cmds = append(cmds, cmd) + } + + { + mdl, cmd := m.readiness.Update(msg) + m.readiness = mdl.(readinessModel) + cmds = append(cmds, cmd) + } + + { + mdl, cmd := m.pods.Update(msg) + m.pods = mdl.(podsModel) + cmds = append(cmds, cmd) + } + + apply := func() { + res, err := m.Client.Resource(m.server) + if err != nil { + m.finalError = fmt.Errorf("resource client: %w", err) + return + } + m.resource = res + + m.applying = inProgress + cmds = append(cmds, applyCmd(m.Ctx, m.resource, m.server.DeepCopy())) + } + + switch msg := msg.(type) { + case manifestSelectedMsg: + m.Namespace.Set(msg.obj) + m.server = msg.obj.(*apiv1.Server) + apply() + + case appliedMsg: + m.applying = completed + m.server = msg.Object.(*apiv1.Server) + + m.readiness.Object = m.server + m.readiness.Resource = m.resource + + m.pods.Object = m.server + m.pods.Resource = m.resource + + cmds = append(cmds, + m.readiness.Init(), + m.pods.Init(), + ) + + case tea.KeyMsg: + log.Println("Received key msg:", msg.String()) + if m.quitting { + switch msg.String() { + case "esc": + if m.finalError == nil { + m.quitting = false + } + + case "l": + cmds = append(cmds, tea.Quit) + // case "s": + // cmds = append(cmds, suspendCmd(context.Background(), m.resource, m.server)) + case "d": + cmds = append(cmds, deleteCmd(context.Background(), m.resource, m.server)) + } + } else { + if msg.String() == "q" { + m.quitting = true + } + } + + // case suspendedMsg: + // if msg.error != nil { + // m.finalError = msg.error + // } else { + // m.goodbye = "Server suspended." + // } + // cmds = append(cmds, tea.Quit) + + case deletedMsg: + if msg.error != nil { + m.finalError = msg.error + } else { + m.goodbye = "Server deleted." + } + cmds = append(cmds, tea.Quit) + + case objectReadyMsg: + m.server = msg.Object.(*apiv1.Server) + // TODO: What to do? + // cmds = append(cmds) // TODO: Port-forward to Pod. + // portForwardCmd(m.Ctx, m.Client, client.PodForNotebook(m.Server)), + // + case podWatchMsg: + if m.readyPod != nil { + break + } + if msg.Pod.Labels == nil || msg.Pod.Labels["role"] != "run" { + break + } + + var ready bool + for _, c := range msg.Pod.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + ready = true + break + } + } + + if ready { + m.readyPod = msg.Pod.DeepCopy() + cmds = append(cmds, + portForwardCmd(m.Ctx, m.Client, + types.NamespacedName{Namespace: m.readyPod.Namespace, Name: m.readyPod.Name}, + client.ForwardedPorts{Local: 8000, Pod: 8080}, + ), + ) + } + + case portForwardReadyMsg: + cmds = append(cmds, serverOpenInBrowser(m.server.DeepCopy())) + + case localURLMsg: + m.localURL = string(msg) + + case tea.WindowSizeMsg: + m.Style.Width(msg.Width) + innerWidth := m.Style.GetWidth() - m.Style.GetHorizontalPadding() + m.readiness.Style = lipgloss.NewStyle().Width(innerWidth) + m.pods.SetStyle(logStyle.Width(innerWidth)) + + case error: + log.Printf("Error message: %v", msg) + m.finalError = msg + m.quitting = true + } + + return m, tea.Batch(cmds...) +} + +// View returns a string based on data in the model. That string which will be +// rendered to the terminal. +func (m ServeModel) View() (v string) { + defer func() { + v = m.Style.Render(v) + }() + + if m.finalError != nil { + v += errorStyle.Width(m.Style.GetWidth()-m.Style.GetHorizontalMargins()-10).Render("Error: "+m.finalError.Error()) + "\n" + // v += helpStyle("Press \"s\" to suspend, \"d\" to delete") + v += helpStyle("Press \"l\" to leave be, \"d\" to delete") + return + } + + if m.goodbye != "" { + v += m.goodbye + "\n" + return + } + + if m.quitting { + v += "Quitting...\n" + v += helpStyle("Press \"l\" to leave be, \"d\" to delete, \"ESC\" to cancel") + // v += helpStyle("Press \"s\" to suspend, \"d\" to delete, \"ESC\" to cancel") + // v += helpStyle("Press \"l\" to leave be, \"s\" to suspend, \"d\" to delete, \"ESC\" to cancel") + return + } + + if m.applying == inProgress { + v += "Applying...\n\n" + } + + v += m.manifests.View() + v += m.readiness.View() + v += m.pods.View() + + if m.portForwarding == inProgress { + v += "Port-forwarding...\n" + } + + if m.localURL != "" && m.portForwarding == inProgress { + v += "\n" + v += fmt.Sprintf("Server URL: %v\n", m.localURL) + } + + v += helpStyle("Press \"q\" to quit") + + return v +} + +func serverOpenInBrowser(s *apiv1.Server) tea.Cmd { + return func() tea.Msg { + url := "http://localhost:8000" + log.Printf("Opening browser to %s\n", url) + browser.OpenURL(url) + return localURLMsg(url) + } +} diff --git a/internal/tui/styles.go b/internal/tui/styles.go new file mode 100644 index 00000000..c169d921 --- /dev/null +++ b/internal/tui/styles.go @@ -0,0 +1,33 @@ +package tui + +import "github.com/charmbracelet/lipgloss" + +var ( + appStyle = lipgloss.NewStyle(). + PaddingTop(1). + PaddingRight(2). + PaddingBottom(1). + PaddingLeft(2) + + podStyle = lipgloss.NewStyle().PaddingLeft(2).Render + + // https://coolors.co/palette/264653-2a9d8f-e9c46a-f4a261-e76f51 + // + logStyle = lipgloss.NewStyle().PaddingLeft(1).Border(lipgloss.NormalBorder(), false, false, false, true) /*lipgloss.Border{ + TopLeft: "| ", + BottomLeft: "| ", + Left: "| ", + })*/ + + helpStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#626262")). + MarginTop(1). + MarginBottom(1). + Render + + errorStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#e76f51")) + + activeSpinnerStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#E9C46A")) + checkMark = lipgloss.NewStyle().Foreground(lipgloss.Color("#2a9d8f")).SetString("✓") + // TODO: Better X mark? + xMark = lipgloss.NewStyle().Foreground(lipgloss.Color("#e76f51")).SetString("x") +) diff --git a/internal/tui/upload.go b/internal/tui/upload.go new file mode 100644 index 00000000..61f8b4f3 --- /dev/null +++ b/internal/tui/upload.go @@ -0,0 +1,169 @@ +package tui + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/charmbracelet/bubbles/progress" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + + "github.com/substratusai/substratus/internal/client" +) + +type uploadModel struct { + // Cancellation + Ctx context.Context + + // Config + Path string + + // Clients + Client client.Interface + Resource *client.Resource + + Mode uploadMode + applying status + creating status + + // Original Object (could be a Dataset, Model, or Server) + Object client.Object + + // Tarring + tarring status + tarredFileCount int + tarball *client.Tarball + + // Uploading + uploading status + uploadProgress progress.Model + + Style lipgloss.Style +} + +type uploadMode int + +const ( + uploadModeCreate = 0 + uploadModeApply = 1 +) + +func (m uploadModel) kind() string { + return m.Object.GetObjectKind().GroupVersionKind().Kind +} + +func (m uploadModel) cleanup() { + log.Println("Cleaning up") + os.Remove(m.tarball.TempDir) +} + +// New initializes all internal fields. +func (m *uploadModel) New() uploadModel { + m.Style = lipgloss.NewStyle() + m.uploadProgress = progress.New(progress.WithDefaultGradient()) + return *m +} + +func (m uploadModel) Active() bool { + return true + //if m.Mode == uploadModeApply { + // return m.applying != completed + //} else if m.Mode == uploadModeCreate { + // return m.creating != completed + //} else { + // panic("Unrecognized mode") + //} +} + +func (m uploadModel) Init() tea.Cmd { + return tea.Sequence( + func() tea.Msg { return uploadInitMsg{} }, + prepareTarballCmd(m.Ctx, m.Path), + ) +} + +type uploadInitMsg struct{} + +func (m uploadModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case uploadInitMsg: + m.tarring = inProgress + + case fileTarredMsg: + m.tarredFileCount++ + return m, nil + + case tarballCompleteMsg: + m.tarring = completed + m.tarball = msg + if m.Mode == uploadModeApply { + m.applying = inProgress + return m, applyWithUploadCmd(m.Ctx, m.Resource, m.Object.DeepCopyObject().(client.Object), m.tarball) + } else if m.Mode == uploadModeCreate { + m.creating = inProgress + return m, createWithUploadCmd(m.Ctx, m.Resource, m.Object.DeepCopyObject().(client.Object), m.tarball) + } else { + panic("unkown upload mode") + } + + case appliedWithUploadMsg: + m.Object = msg.Object + m.applying = completed + m.uploading = inProgress + return m, uploadTarballCmd(m.Ctx, m.Resource, m.Object.DeepCopyObject().(client.Object), m.tarball) + + case createdWithUploadMsg: + m.Object = msg.Object + m.creating = completed + m.uploading = inProgress + return m, uploadTarballCmd(m.Ctx, m.Resource, m.Object.DeepCopyObject().(client.Object), m.tarball) + + case uploadTarballProgressMsg: + return m, m.uploadProgress.SetPercent(float64(msg)) + + case tarballUploadedMsg: + m.uploading = completed + m.Object = msg.Object + + // FrameMsg is sent when the progress bar wants to animate itself + case progress.FrameMsg: + progressModel, cmd := m.uploadProgress.Update(msg) + m.uploadProgress = progressModel.(progress.Model) + return m, cmd + } + + return m, nil +} + +// View returns a string based on data in the model. That string which will be +// rendered to the terminal. +func (m uploadModel) View() (v string) { + defer func() { + if v != "" { + v = m.Style.Render(v) + } + }() + m.uploadProgress.Width = m.Style.GetWidth() + + if m.tarring == inProgress { + v += "Tarring...\n" + v += fmt.Sprintf("File count: %v\n", m.tarredFileCount) + } + + if m.applying == inProgress { + v += "Applying...\n" + } + + if m.creating == inProgress { + v += "Creating...\n" + } + + if m.uploading == inProgress { + v += "Uploading...\n\n" + v += m.uploadProgress.View() + "\n\n" + } + + return v +} diff --git a/kubectl/cmd/applybuild/main.go b/kubectl/cmd/applybuild/main.go deleted file mode 100644 index 93c2dd38..00000000 --- a/kubectl/cmd/applybuild/main.go +++ /dev/null @@ -1,15 +0,0 @@ -package main - -import ( - "github.com/substratusai/substratus/kubectl/internal/commands" - "k8s.io/klog/v2" -) - -var Version = "development" - -func main() { - commands.Version = Version - if err := commands.ApplyBuild().Execute(); err != nil { - klog.Fatal(err) - } -} diff --git a/kubectl/cmd/notebook/main.go b/kubectl/cmd/notebook/main.go deleted file mode 100644 index 293be1f6..00000000 --- a/kubectl/cmd/notebook/main.go +++ /dev/null @@ -1,15 +0,0 @@ -package main - -import ( - "github.com/substratusai/substratus/kubectl/internal/commands" - "k8s.io/klog/v2" -) - -var Version = "development" - -func main() { - commands.Version = Version - if err := commands.Notebook().Execute(); err != nil { - klog.Fatal(err) - } -} diff --git a/kubectl/internal/client/notebook.go b/kubectl/internal/client/notebook.go deleted file mode 100644 index eb5525b0..00000000 --- a/kubectl/internal/client/notebook.go +++ /dev/null @@ -1,84 +0,0 @@ -package client - -import ( - "fmt" - - apiv1 "github.com/substratusai/substratus/api/v1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func NotebookForObject(obj Object) (*apiv1.Notebook, error) { - var nb *apiv1.Notebook - - switch obj := obj.DeepCopyObject().(type) { - case *apiv1.Notebook: - nb = obj - - case *apiv1.Model: - return nil, fmt.Errorf("notebooks for models are not yet supported") - /* - nb = &apiv1.Notebook{ - ObjectMeta: metav1.ObjectMeta{ - Name: obj.Name, - Namespace: obj.Namespace, - }, - Spec: apiv1.NotebookSpec{ - // TODO: How to map base model / saved model to notebook mounts? - Image: obj.Spec.Image, - Params: obj.Spec.Params, - }, - } - */ - - case *apiv1.Server: - return nil, fmt.Errorf("notebooks for servers are not yet supported") - /* - nb = &apiv1.Notebook{ - ObjectMeta: metav1.ObjectMeta{ - Name: obj.Name, - Namespace: obj.Namespace, - }, - Spec: apiv1.NotebookSpec{ - Image: obj.Spec.Image, - Model: &obj.Spec.Model, - }, - } - */ - - case *apiv1.Dataset: - return nil, fmt.Errorf("notebooks for datasets are not yet supported") - // NOTE: For this to work for Dataset development purposes, the Notebook - // controllers needs to mount a directory to receive the dataset. - // (/content/data). - /* - nb = &apiv1.Notebook{ - ObjectMeta: metav1.ObjectMeta{ - Name: obj.Name, - Namespace: obj.Namespace, - }, - Spec: apiv1.NotebookSpec{ - Build: obj.Spec.Build, - //Dataset: &apiv1.ObjectRef{ - // Name: obj.Name, - //}, - Params: obj.Spec.Params, - }, - } - */ - - default: - return nil, fmt.Errorf("unknown object type: %T", obj) - } - - // "This field is managed by the API server and should not be changed by the user." - // https://kubernetes.io/docs/reference/using-api/server-side-apply/#field-management - nb.ObjectMeta.ManagedFields = nil - - nb.TypeMeta = metav1.TypeMeta{ - APIVersion: "substratus.ai/v1", - Kind: "Notebook", - } - - return nb, nil -} diff --git a/kubectl/internal/commands/applybuild.go b/kubectl/internal/commands/applybuild.go deleted file mode 100644 index 78d9492d..00000000 --- a/kubectl/internal/commands/applybuild.go +++ /dev/null @@ -1,129 +0,0 @@ -package commands - -import ( - "flag" - "fmt" - "os" - - "github.com/spf13/cobra" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" - - "github.com/substratusai/substratus/kubectl/internal/client" -) - -func ApplyBuild() *cobra.Command { - var flags struct { - namespace string - filename string - build string - kubeconfig string - forceConflicts bool - } - - cmd := &cobra.Command{ - Use: "applybuild [flags] BUILD_CONTEXT", - Args: cobra.ExactArgs(1), - Short: "Apply a Substratus object, upload and build container in-cluster from a local directory", - Version: Version, - RunE: func(cmd *cobra.Command, args []string) error { - client.Version = Version - - ctx := cmd.Context() - - if flags.filename == "" { - return fmt.Errorf("-f (--filename) is required") - } - flags.build = args[0] - - tarball, err := client.PrepareImageTarball(ctx, flags.build) - if err != nil { - return fmt.Errorf("preparing tarball: %w", err) - } - defer os.Remove(tarball.TempDir) - - kubeconfigNamespace, restConfig, err := buildConfigFromFlags("", flags.kubeconfig) - if err != nil { - return fmt.Errorf("rest config: %w", err) - } - - namespace := "default" - if flags.namespace != "" { - namespace = flags.namespace - } else if kubeconfigNamespace != "" { - namespace = kubeconfigNamespace - } - - clientset, err := kubernetes.NewForConfig(restConfig) - if err != nil { - return fmt.Errorf("clientset: %w", err) - } - - manifest, err := os.ReadFile(flags.filename) - if err != nil { - return fmt.Errorf("reading file: %w", err) - } - - obj, err := client.Decode(manifest) - if err != nil { - return fmt.Errorf("decoding: %w", err) - } - - if obj.GetNamespace() == "" { - // When there is no .metadata.namespace set in the manifest... - obj.SetNamespace(namespace) - } else { - // TODO: Closer match kubectl behavior here by differentiaing between - // the short -n and long --namespace flags. - // See example kubectl error: - // error: the namespace from the provided object "a" does not match the namespace "b". You must pass '--namespace=a' to perform this operation. - if flags.namespace != "" && flags.namespace != obj.GetNamespace() { - // When there is .metadata.namespace set in the manifest and - // a conflicting -n or --namespace flag... - return fmt.Errorf("the namespace from the provided object %q does not match the namespace %q from flag", obj.GetNamespace(), flags.namespace) - } - } - - c := NewClient(clientset, restConfig) - r, err := c.Resource(obj) - if err != nil { - return fmt.Errorf("resource client: %w", err) - } - - if err := client.SetUploadContainerSpec(obj, tarball, NewUUID()); err != nil { - return fmt.Errorf("setting upload in spec: %w", err) - } - if err := client.ClearImage(obj); err != nil { - return fmt.Errorf("clearing image: %w", err) - } - - if err := r.Apply(obj, flags.forceConflicts); err != nil { - return fmt.Errorf("applying: %w", err) - } - - if err := r.Upload(ctx, obj, tarball); err != nil { - return fmt.Errorf("uploading: %w", err) - } - - return nil - }, - } - - defaultKubeconfig := os.Getenv("KUBECONFIG") - if defaultKubeconfig == "" { - defaultKubeconfig = clientcmd.RecommendedHomeFile - } - cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "") - cmd.Flags().StringVarP(&flags.filename, "filename", "f", "", "Filename identifying the resource to apply and build.") - cmd.Flags().BoolVar(&flags.forceConflicts, "force-conflicts", false, "If true, server-side apply will force the changes against conflicts.") - - cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook") - - // Add standard kubectl logging flags (for example: -v=2). - goflags := flag.NewFlagSet("", flag.PanicOnError) - klog.InitFlags(goflags) - cmd.Flags().AddGoFlagSet(goflags) - - return cmd -} diff --git a/kubectl/internal/commands/applybuild_test.go b/kubectl/internal/commands/applybuild_test.go deleted file mode 100644 index e7f575ed..00000000 --- a/kubectl/internal/commands/applybuild_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package commands_test - -import ( - "net/http" - "net/http/httptest" - "sync" - "testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - apiv1 "github.com/substratusai/substratus/api/v1" - "github.com/substratusai/substratus/kubectl/internal/commands" - "k8s.io/apimachinery/pkg/types" -) - -func TestApplyBuild(t *testing.T) { - cmd := commands.ApplyBuild() - cmd.SetArgs([]string{ - "--filename", "./test-applybuild/notebook.yaml", - "--kubeconfig", kubectlKubeconfigPath, - //"-v=9", - "./test-applybuild", - }) - var wg sync.WaitGroup - - wg.Add(1) - go func() { - defer wg.Done() - if err := cmd.Execute(); err != nil { - t.Error(err) - } - }() - - var uploadedPath string - var uploadedPathMtx sync.Mutex - mockBucketServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - t.Log("mockBucketServer handler called") - - uploadedPathMtx.Lock() - uploadedPath = r.URL.String() - uploadedPathMtx.Unlock() - })) - defer mockBucketServer.Close() - - nb := &apiv1.Notebook{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-applybuild", - Namespace: "default", - }, - } - require.EventuallyWithT(t, func(t *assert.CollectT) { - err := k8sClient.Get(ctx, types.NamespacedName{Namespace: nb.Namespace, Name: nb.Name}, nb) - assert.NoError(t, err, "getting notebook") - }, timeout, interval, "waiting for the notebook to be created") - - nb.Status.BuildUpload = apiv1.UploadStatus{ - SignedURL: mockBucketServer.URL + "/some-signed-url", - RequestID: testUUID, - } - require.NoError(t, k8sClient.Status().Update(ctx, nb)) - - require.EventuallyWithT(t, func(t *assert.CollectT) { - uploadedPathMtx.Lock() - assert.Equal(t, "/some-signed-url", uploadedPath) - uploadedPathMtx.Unlock() - }, timeout, interval, "waiting for command to upload the tarball") - - t.Log("wait group waiting") - wg.Wait() -} diff --git a/kubectl/internal/commands/main_test.go b/kubectl/internal/commands/main_test.go deleted file mode 100644 index 41f55b7c..00000000 --- a/kubectl/internal/commands/main_test.go +++ /dev/null @@ -1,146 +0,0 @@ -package commands_test - -import ( - "bytes" - "context" - "fmt" - "log" - "os" - "path/filepath" - "testing" - "time" - - ctrl "sigs.k8s.io/controller-runtime" - - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - apiv1 "github.com/substratusai/substratus/api/v1" - iclient "github.com/substratusai/substratus/kubectl/internal/client" - "github.com/substratusai/substratus/kubectl/internal/commands" - //+kubebuilder:scaffold:imports -) - -const ( - timeout = time.Second * 5 - interval = time.Second / 10 - testUUID = "c1d1eb65-75bd-48a5-9bad-802810fc9117" -) - -var ( - kubectlKubeconfigPath string - k8sClient client.Client - testEnv *envtest.Environment - ctx context.Context - cancel context.CancelFunc - stdout bytes.Buffer -) - -func TestMain(m *testing.M) { - commands.NewClient = newClientWithMockPortForward - commands.NotebookStdout = &stdout - commands.NewUUID = func() string { return testUUID } - - //var buf bytes.Buffer - logf.SetLogger(zap.New( - zap.UseDevMode(true), - //zap.WriteTo(&buf), - )) - - ctx, cancel = context.WithCancel(context.TODO()) - - log.Println("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - } - - cfg, err := testEnv.Start() - requireNoError(err) - - kubectlUser, err := testEnv.ControlPlane.AddUser(envtest.User{ - Name: "kubectl-user", - Groups: []string{"system:masters"}, - }, nil) - requireNoError(err) - kubeconfig, err := kubectlUser.KubeConfig() - requireNoError(err) - kubectlKubeconfigPath = filepath.Join(os.TempDir(), "kubeconfig.yaml") - requireNoError(os.WriteFile(kubectlKubeconfigPath, kubeconfig, 0644)) - - log.Printf("wrote test kubeconfig to: %s", kubectlKubeconfigPath) - - requireNoError(apiv1.AddToScheme(scheme.Scheme)) - - //+kubebuilder:scaffold:scheme - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - requireNoError(err) - - mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme.Scheme, - MetricsBindAddress: "0", - }) - requireNoError(err) - - ctx, cancel := context.WithCancel(ctx) - - go func() { - log.Println("starting manager") - err := mgr.Start(ctx) - if err != nil { - log.Printf("starting manager: %s", err) - } - }() - - log.Println("running tests") - code := m.Run() - - // TODO: Run cleanup on ctrl-C, etc. - log.Println("stopping manager") - cancel() - log.Println("stopping test environment") - requireNoError(testEnv.Stop()) - - fmt.Printf(` -======== Command Stdout ======== -%s -================================ -`, stdout.String()) - - os.Exit(code) -} - -func requireNoError(err error) { - if err != nil { - log.Fatal(err) - } -} - -type mockClient struct { - iclient.Interface -} - -func (c *mockClient) PortForwardNotebook(ctx context.Context, verbose bool, nb *apiv1.Notebook, ready chan struct{}) error { - log.Println("mockClient.PortForwardNotebook called") - select { - case ready <- struct{}{}: - fmt.Println("sent ready") - default: - fmt.Println("no ready sent") - } - ctx.Done() - return fmt.Errorf("mock PortForwardNotebook exiting because of ctx.Done()") -} - -func newClientWithMockPortForward(inf kubernetes.Interface, cfg *rest.Config) iclient.Interface { - return &mockClient{ - // Only mock PortForwardNotebook() - Interface: iclient.NewClient(inf, cfg), - } -} diff --git a/kubectl/internal/commands/notebook.go b/kubectl/internal/commands/notebook.go deleted file mode 100644 index 4e7f60ef..00000000 --- a/kubectl/internal/commands/notebook.go +++ /dev/null @@ -1,352 +0,0 @@ -package commands - -import ( - "context" - "errors" - "flag" - "fmt" - "math" - "os" - "os/signal" - "path/filepath" - "sync" - "syscall" - "time" - - "github.com/briandowns/spinner" - "github.com/pkg/browser" - "github.com/spf13/cobra" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" - "k8s.io/utils/ptr" - - apiv1 "github.com/substratusai/substratus/api/v1" - "github.com/substratusai/substratus/kubectl/internal/client" -) - -func Notebook() *cobra.Command { - var flags struct { - dir string - build string - kubeconfig string - filename string - namespace string - noOpenBrowser bool - sync bool - forceConflicts bool - noSuspend bool - timeout time.Duration - } - - cmd := &cobra.Command{ - Use: "notebook [flags] NAME", - Short: "Start a Jupyter Notebook development environment", - Args: cobra.MaximumNArgs(1), - Version: Version, - RunE: func(cmd *cobra.Command, args []string) error { - client.Version = Version - - ctx, ctxCancel := context.WithCancel(cmd.Context()) - cancel := func() { - klog.V(1).Info("Context cancelled") - ctxCancel() - } - defer cancel() - - // The -v flag is managed by klog, so we need to check it manually. - var verbose bool - if cmd.Flag("v").Changed { - verbose = true - } - - if flags.dir != "" { - if flags.build == "" { - flags.build = flags.dir - } - // If the user specified a directory, we assume they want to sync - // unless they explicitly set --sync themselves. - if !cmd.Flag("sync").Changed { - flags.sync = true - } - // If the user specified a directory, we assume they have a notebook.yaml - // file in their directory unless they explicitly set --filename themselves. - if !cmd.Flag("filename").Changed { - flags.filename = filepath.Join(flags.dir, "notebook.yaml") - } - } - - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - go func() { - <-sigs - cancel() - }() - - spin := spinner.New(spinner.CharSets[9], 100*time.Millisecond) - - kubeconfigNamespace, restConfig, err := buildConfigFromFlags("", flags.kubeconfig) - if err != nil { - return fmt.Errorf("rest config: %w", err) - } - - namespace := "default" - if flags.namespace != "" { - namespace = flags.namespace - } else if kubeconfigNamespace != "" { - namespace = kubeconfigNamespace - } - - clientset, err := kubernetes.NewForConfig(restConfig) - if err != nil { - return fmt.Errorf("clientset: %w", err) - } - - c := NewClient(clientset, restConfig) - notebooks, err := c.Resource(&apiv1.Notebook{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "substratus.ai/v1", - Kind: "Notebook", - }, - }) - if err != nil { - return fmt.Errorf("resource client: %w", err) - } - - var obj client.Object - if len(args) == 1 { - fetched, err := notebooks.Get(namespace, args[0]) - if err != nil { - return fmt.Errorf("getting notebook: %w", err) - } - obj = fetched.(client.Object) - } else if flags.filename != "" { - manifest, err := os.ReadFile(flags.filename) - if err != nil { - return fmt.Errorf("reading file: %w", err) - } - obj, err = client.Decode(manifest) - if err != nil { - return fmt.Errorf("decoding: %w", err) - } - if obj.GetNamespace() == "" { - // When there is no .metadata.namespace set in the manifest... - obj.SetNamespace(namespace) - } else { - // TODO: Closer match kubectl behavior here by differentiaing between - // the short -n and long --namespace flags. - // See example kubectl error: - // error: the namespace from the provided object "a" does not match the namespace "b". You must pass '--namespace=a' to perform this operation. - if flags.namespace != "" && flags.namespace != obj.GetNamespace() { - // When there is .metadata.namespace set in the manifest and - // a conflicting -n or --namespace flag... - return fmt.Errorf("the namespace from the provided object %q does not match the namespace %q from flag", obj.GetNamespace(), flags.namespace) - } - } - } else { - return fmt.Errorf("must specify -f (--filename) flag or NAME argument") - } - - nb, err := client.NotebookForObject(obj) - if err != nil { - return fmt.Errorf("notebook for object: %w", err) - } - nb.Spec.Suspend = ptr.To(false) - - var tarball *client.Tarball - if flags.build != "" { - spin.Suffix = " Preparing tarball..." - spin.Start() - - var err error - tarball, err = client.PrepareImageTarball(ctx, flags.build) - if err != nil { - return fmt.Errorf("preparing tarball: %w", err) - } - defer os.Remove(tarball.TempDir) - - spin.Stop() - fmt.Fprintln(NotebookStdout, "Tarball prepared.") - - if err := client.ClearImage(nb); err != nil { - return fmt.Errorf("clearing image in spec: %w", err) - } - if err := client.SetUploadContainerSpec(nb, tarball, NewUUID()); err != nil { - return fmt.Errorf("setting upload in spec: %w", err) - } - } - - if err := notebooks.Apply(nb, flags.forceConflicts); err != nil { - return fmt.Errorf("applying: %w", err) - } - - cleanup := func() { - // Use a new context to avoid using the cancelled one. - // ctx := context.Background() - - if flags.noSuspend { - fmt.Fprintln(NotebookStdout, "Skipping notebook suspension, it will keep running.") - } else { - // Suspend notebook. - spin.Suffix = " Suspending notebook..." - spin.Start() - _, err := notebooks.Patch(nb.Namespace, nb.Name, types.MergePatchType, []byte(`{"spec": {"suspend": true} }`), &metav1.PatchOptions{}) - spin.Stop() - if err != nil { - klog.Errorf("Error suspending notebook: %v", err) - } else { - fmt.Fprintln(NotebookStdout, "Notebook suspended.") - } - } - } - defer cleanup() - - if flags.build != "" { - spin.Suffix = " Uploading tarball..." - spin.Start() - - if err := notebooks.Upload(ctx, nb, tarball); err != nil { - return fmt.Errorf("uploading: %w", err) - } - - spin.Stop() - fmt.Fprintln(NotebookStdout, "Tarball uploaded.") - } - - spin.Suffix = " Waiting for Notebook to be ready..." - spin.Start() - - waitReadyCtx, cancelWaitReady := context.WithTimeout(ctx, flags.timeout) - defer cancelWaitReady() // Avoid context leak. - if err := notebooks.WaitReady(waitReadyCtx, nb); err != nil { - return fmt.Errorf("waiting for notebook to be ready: %w", err) - } - - spin.Stop() - fmt.Fprintln(NotebookStdout, "Notebook ready.") - - var wg sync.WaitGroup - - if flags.sync { - wg.Add(1) - go func() { - defer func() { - wg.Done() - klog.V(2).Info("Syncing files from notebook: Done.") - // Stop other goroutines. - cancel() - }() - if err := c.SyncFilesFromNotebook(ctx, nb, flags.build); err != nil { - if !errors.Is(err, context.Canceled) { - klog.Errorf("Error syncing files from notebook: %v", err) - } - } - }() - } - - serveReady := make(chan struct{}) - wg.Add(1) - go func() { - defer func() { - wg.Done() - klog.V(2).Info("Port-forwarding: Done.") - // Stop other goroutines. - cancel() - }() - - const maxRetries = 3 - for i := 0; i < maxRetries; i++ { - portFwdCtx, cancelPortFwd := context.WithCancel(ctx) - defer cancelPortFwd() // Avoid a context leak - runtime.ErrorHandlers = []func(err error){ - func(err error) { - // Cancel a broken port forward to attempt to restart the port-forward. - klog.Errorf("Port-forward error: %v", err) - cancelPortFwd() - }, - } - - // portForward will close the ready channel when it returns. - // so we only use the outer ready channel once. On restart of the portForward, - // we use a new channel. - var ready chan struct{} - if i == 0 { - ready = serveReady - } else { - ready = make(chan struct{}) - } - - if err := c.PortForwardNotebook(portFwdCtx, verbose, nb, ready); err != nil { - klog.Errorf("Port-forward returned an error: %v", err) - } - - // Check if the command's context is cancelled, if so, - // avoid restarting the port forward. - if err := ctx.Err(); err != nil { - klog.V(1).Infof("Context done, not attempting to restart port-forward: %v", err.Error()) - return - } - - cancelPortFwd() // Avoid a build up of contexts before returning. - backoff := time.Duration(math.Pow(2, float64(i))) * time.Second - klog.V(1).Infof("Restarting port forward (index = %v), after backoff: %s", i, backoff) - time.Sleep(backoff) - } - klog.V(1).Info("Done trying to port-forward") - }() - - spin.Suffix = " Waiting for connection to be ready to serve..." - spin.Start() - select { - case <-serveReady: - case <-ctx.Done(): - return fmt.Errorf("context done while waiting on connection to be ready: %w", ctx.Err()) - } - spin.Stop() - fmt.Fprintln(NotebookStdout, "Connection ready.") - - // TODO(nstogner): Grab token from Notebook status. - url := "http://localhost:8888?token=default" - if !flags.noOpenBrowser { - fmt.Fprintf(NotebookStdout, "Opening browser to %s\n", url) - browser.OpenURL(url) - } else { - fmt.Fprintf(NotebookStdout, "Open browser to: %s\n", url) - } - - klog.V(2).Info("Waiting for routines to complete before exiting") - wg.Wait() - klog.V(2).Info("Routines completed, exiting") - - return nil - }, - } - - defaultKubeconfig := os.Getenv("KUBECONFIG") - if defaultKubeconfig == "" { - defaultKubeconfig = clientcmd.RecommendedHomeFile - } - cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "") - - cmd.Flags().StringVarP(&flags.dir, "dir", "d", "", "Directory to launch the Notebook for. Equivalent to -f /notebook.yaml -b -s") - cmd.Flags().StringVarP(&flags.build, "build", "b", "", "Build the Notebook from this local directory") - cmd.Flags().StringVarP(&flags.filename, "filename", "f", "", "Filename identifying the resource to develop against.") - cmd.Flags().BoolVarP(&flags.sync, "sync", "s", false, "Sync local directory with Notebook") - - cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook") - - cmd.Flags().BoolVar(&flags.noSuspend, "no-suspend", false, "Do not suspend the Notebook when exiting") - cmd.Flags().BoolVar(&flags.forceConflicts, "force-conflicts", true, "If true, server-side apply will force the changes against conflicts.") - cmd.Flags().BoolVar(&flags.noOpenBrowser, "no-open-browser", false, "Do not open the Notebook in a browser") - cmd.Flags().DurationVarP(&flags.timeout, "timeout", "t", 20*time.Minute, "Timeout for Notebook to become ready") - - // Add standard kubectl logging flags (for example: -v=2). - goflags := flag.NewFlagSet("", flag.PanicOnError) - klog.InitFlags(goflags) - cmd.Flags().AddGoFlagSet(goflags) - - return cmd -} diff --git a/kubectl/internal/commands/notebook_test.go b/kubectl/internal/commands/notebook_test.go deleted file mode 100644 index 46937097..00000000 --- a/kubectl/internal/commands/notebook_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package commands_test - -import ( - "context" - "net/http" - "net/http/httptest" - "sync" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - apiv1 "github.com/substratusai/substratus/api/v1" - "github.com/substratusai/substratus/kubectl/internal/commands" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" -) - -func TestNotebook(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - cmd := commands.Notebook() - cmd.SetArgs([]string{ - "--filename", "./test-notebook/notebook.yaml", - "--build", "./test-notebook", - "--kubeconfig", kubectlKubeconfigPath, - "--no-open-browser", - "-v=4", - }) - cmd.SetContext(ctx) - var wg sync.WaitGroup - - wg.Add(1) - go func() { - defer wg.Done() - if err := cmd.Execute(); err != nil { - t.Error(err) - } - }() - - var uploadedPath string - var uploadedPathMtx sync.Mutex - mockBucketServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - t.Log("mockBucketServer handler called") - - uploadedPathMtx.Lock() - uploadedPath = r.URL.String() - uploadedPathMtx.Unlock() - })) - defer mockBucketServer.Close() - - nb := &apiv1.Notebook{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-notebook", - Namespace: "default", - }, - } - require.EventuallyWithT(t, func(t *assert.CollectT) { - err := k8sClient.Get(ctx, types.NamespacedName{Namespace: nb.Namespace, Name: nb.Name}, nb) - assert.NoError(t, err, "getting notebook") - }, timeout, interval, "waiting for the notebook to be created") - - nb.Status.BuildUpload = apiv1.UploadStatus{ - SignedURL: mockBucketServer.URL + "/some-signed-url", - RequestID: testUUID, - } - require.NoError(t, k8sClient.Status().Update(ctx, nb)) - - require.EventuallyWithT(t, func(t *assert.CollectT) { - uploadedPathMtx.Lock() - assert.Equal(t, "/some-signed-url", uploadedPath) - uploadedPathMtx.Unlock() - }, timeout, interval, "waiting for command to upload the tarball") - - require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Namespace: nb.Namespace, Name: nb.Name}, nb)) - nb.Status.Ready = true - require.NoError(t, k8sClient.Status().Update(ctx, nb)) - - require.EventuallyWithT(t, func(t *assert.CollectT) { - assert.Contains(t, stdout.String(), "Open browser to") - }, timeout, interval, "waiting for command to indicate a browser should be opened") - - t.Logf("Killing command") - cancel() - - t.Log("Test wait group waiting") - wg.Wait() - - // Use context.Background() because the original context is cancelled. - require.NoError(t, k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nb.Namespace, Name: nb.Name}, nb)) - require.True(t, nb.IsSuspended(), "Make sure cleanup ran") -} diff --git a/kubectl/internal/commands/test-applybuild/Dockerfile b/kubectl/internal/commands/test-applybuild/Dockerfile deleted file mode 100644 index 64f6dbed..00000000 --- a/kubectl/internal/commands/test-applybuild/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM substratusai/base - -RUN touch touch.txt - -COPY . /tmp -#COPY hello.txt hello.txt diff --git a/kubectl/internal/commands/test-applybuild/hello.txt b/kubectl/internal/commands/test-applybuild/hello.txt deleted file mode 100644 index 5ab2f8a4..00000000 --- a/kubectl/internal/commands/test-applybuild/hello.txt +++ /dev/null @@ -1 +0,0 @@ -Hello \ No newline at end of file diff --git a/kubectl/internal/commands/test-applybuild/notebook.yaml b/kubectl/internal/commands/test-applybuild/notebook.yaml deleted file mode 100644 index 07bb20b4..00000000 --- a/kubectl/internal/commands/test-applybuild/notebook.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: substratus.ai/v1 -kind: Notebook -metadata: - name: test-applybuild -spec: - image: substratusai/base:latest - params: - foo: bar - x: 123 diff --git a/kubectl/internal/commands/test-notebook/Dockerfile b/kubectl/internal/commands/test-notebook/Dockerfile deleted file mode 100644 index 64f6dbed..00000000 --- a/kubectl/internal/commands/test-notebook/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM substratusai/base - -RUN touch touch.txt - -COPY . /tmp -#COPY hello.txt hello.txt diff --git a/kubectl/internal/commands/test-notebook/hello.txt b/kubectl/internal/commands/test-notebook/hello.txt deleted file mode 100644 index 5ab2f8a4..00000000 --- a/kubectl/internal/commands/test-notebook/hello.txt +++ /dev/null @@ -1 +0,0 @@ -Hello \ No newline at end of file diff --git a/kubectl/internal/commands/test-notebook/notebook.yaml b/kubectl/internal/commands/test-notebook/notebook.yaml deleted file mode 100644 index 742b310d..00000000 --- a/kubectl/internal/commands/test-notebook/notebook.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: substratus.ai/v1 -kind: Notebook -metadata: - name: test-notebook -spec: - image: substratusai/base:main - params: - foo: bar - x: 123