diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index c58a8ac6..ccff10e2 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -8,4 +8,8 @@ jobs:
steps:
- name: Check out repository code
uses: actions/checkout@v3
+ - name: Setup Go 1.21.x
+ uses: actions/setup-go@v4
+ with:
+ go-version: '1.21.x'
- run: make test-integration
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
index c6255626..ec31f9fb 100644
--- a/.goreleaser.yaml
+++ b/.goreleaser.yaml
@@ -6,23 +6,9 @@ before:
release:
prerelease: "true"
builds:
- - id: kubectl-applybuild
- binary: kubectl-applybuild
- main: ./kubectl/cmd/applybuild/
- ldflags: "-X 'main.Version={{.Version}}'"
- env:
- - CGO_ENABLED=0
- goos:
- - linux
- - windows
- - darwin
- goarch:
- - amd64
- - arm64
- - arm
- - id: kubectl-notebook
- binary: kubectl-notebook
- main: ./kubectl/cmd/notebook/
+ - id: sub
+ binary: sub
+ main: ./cmd/sub/
ldflags: "-X 'main.Version={{.Version}}'"
env:
- CGO_ENABLED=0
@@ -61,13 +47,12 @@ archives:
format_overrides:
- goos: windows
format: zip
- - id: kubectl-plugins
+ - id: sub
builds:
- - kubectl-applybuild
- - kubectl-notebook
+ - sub
format: tar.gz
name_template: >-
- kubectl-plugins-
+ sub-
{{- .Os }}-
{{- .Arch }}
# use zip for windows archives
diff --git a/Dockerfile b/Dockerfile
index 08b61397..c7145776 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
# Build the manager binary
-FROM golang:1.19 as builder
+FROM golang:1.21 as builder
ARG TARGETOS
ARG TARGETARCH
diff --git a/Dockerfile.sci-gcp b/Dockerfile.sci-gcp
index 5bd6c9c7..28360a2b 100644
--- a/Dockerfile.sci-gcp
+++ b/Dockerfile.sci-gcp
@@ -1,5 +1,5 @@
# Start from the latest go base image
-FROM golang:1.20-bookworm AS builder
+FROM golang:1.21-bookworm AS builder
ARG TARGETOS=linux
ARG TARGETARCH=amd64
diff --git a/Dockerfile.sci-kind b/Dockerfile.sci-kind
index b799c15e..c1e4bfba 100644
--- a/Dockerfile.sci-kind
+++ b/Dockerfile.sci-kind
@@ -1,5 +1,5 @@
# Start from the latest go base image
-FROM golang:1.19 AS builder
+FROM golang:1.21 AS builder
ARG TARGETOS=linux
ARG TARGETARCH=amd64
diff --git a/Makefile b/Makefile
index cefe485b..924cacfa 100644
--- a/Makefile
+++ b/Makefile
@@ -189,7 +189,7 @@ dev-down-aws: build-installer
.PHONY: dev-run-gcp
# Controller manager configuration #
dev-run-gcp: export CLOUD=gcp
-dev-run-gcp: PROJECT_ID ?= $(shell gcloud config get project)
+dev-run-gcp: export PROJECT_ID=$(shell gcloud config get project)
dev-run-gcp: export CLUSTER_NAME=substratus
dev-run-gcp: export CLUSTER_LOCATION=us-central1
dev-run-gcp: export PRINCIPAL=substratus@${PROJECT_ID}.iam.gserviceaccount.com
diff --git a/api/v1/conditions.go b/api/v1/conditions.go
index 0def73dc..3b9bdd9a 100644
--- a/api/v1/conditions.go
+++ b/api/v1/conditions.go
@@ -3,9 +3,8 @@ package v1
const (
ConditionUploaded = "Uploaded"
ConditionBuilt = "Built"
- ConditionLoaded = "Loaded"
- ConditionModelled = "Modelled"
- ConditionDeployed = "Deployed"
+ ConditionComplete = "Complete"
+ ConditionServing = "Serving"
)
const (
@@ -20,6 +19,7 @@ const (
ReasonJobNotComplete = "JobNotComplete"
ReasonJobComplete = "JobComplete"
+ ReasonJobFailed = "JobFailed"
ReasonDeploymentReady = "DeploymentReady"
ReasonDeploymentNotReady = "DeploymentNotReady"
ReasonPodReady = "PodReady"
diff --git a/api/v1/model_types.go b/api/v1/model_types.go
index 9599dba6..97ffef92 100644
--- a/api/v1/model_types.go
+++ b/api/v1/model_types.go
@@ -23,12 +23,12 @@ type ModelSpec struct {
// Resources are the compute resources required by the container.
Resources *Resources `json:"resources,omitempty"`
- // BaseModel should be set in order to mount another model to be
+ // Model should be set in order to mount another model to be
// used for transfer learning.
- BaseModel *ObjectRef `json:"baseModel,omitempty"`
+ Model *ObjectRef `json:"model,omitempty"`
// Dataset to mount for training.
- TrainingDataset *ObjectRef `json:"trainingDataset,omitempty"`
+ Dataset *ObjectRef `json:"dataset,omitempty"`
// Parameters are passing into the model training/loading container as environment variables.
// Environment variable name will be `"PARAM_" + uppercase(key)`.
@@ -42,6 +42,7 @@ func (m *Model) GetParams() map[string]intstr.IntOrString {
func (m *Model) GetBuild() *Build {
return m.Spec.Build
}
+
func (m *Model) SetBuild(b *Build) {
m.Spec.Build = b
}
diff --git a/api/v1/notebook_types.go b/api/v1/notebook_types.go
index c3604acd..88607412 100644
--- a/api/v1/notebook_types.go
+++ b/api/v1/notebook_types.go
@@ -44,6 +44,7 @@ func (n *Notebook) GetParams() map[string]intstr.IntOrString {
func (n *Notebook) GetBuild() *Build {
return n.Spec.Build
}
+
func (n *Notebook) SetBuild(b *Build) {
n.Spec.Build = b
}
@@ -79,6 +80,10 @@ func (n *Notebook) GetStatusUpload() UploadStatus {
return n.Status.BuildUpload
}
+func (n *Notebook) GetStatusArtifacts() ArtifactsStatus {
+ return n.Status.Artifacts
+}
+
func (n *Notebook) IsSuspended() bool {
return n.Spec.Suspend != nil && *n.Spec.Suspend
}
@@ -92,6 +97,9 @@ type NotebookStatus struct {
// Conditions is the list of conditions that describe the current state of the Notebook.
Conditions []metav1.Condition `json:"conditions,omitempty"`
+ // Artifacts status.
+ Artifacts ArtifactsStatus `json:"artifacts,omitempty"`
+
// BuildUpload contains the status of the build context upload.
BuildUpload UploadStatus `json:"buildUpload,omitempty"`
}
diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
index e0123ce5..d86743f3 100644
--- a/api/v1/zz_generated.deepcopy.go
+++ b/api/v1/zz_generated.deepcopy.go
@@ -321,13 +321,13 @@ func (in *ModelSpec) DeepCopyInto(out *ModelSpec) {
*out = new(Resources)
(*in).DeepCopyInto(*out)
}
- if in.BaseModel != nil {
- in, out := &in.BaseModel, &out.BaseModel
+ if in.Model != nil {
+ in, out := &in.Model, &out.Model
*out = new(ObjectRef)
**out = **in
}
- if in.TrainingDataset != nil {
- in, out := &in.TrainingDataset, &out.TrainingDataset
+ if in.Dataset != nil {
+ in, out := &in.Dataset, &out.Dataset
*out = new(ObjectRef)
**out = **in
}
@@ -507,6 +507,7 @@ func (in *NotebookStatus) DeepCopyInto(out *NotebookStatus) {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
+ out.Artifacts = in.Artifacts
in.BuildUpload.DeepCopyInto(&out.BuildUpload)
}
diff --git a/cmd/controllermanager/main.go b/cmd/controllermanager/main.go
index 3d83e191..4e32db46 100644
--- a/cmd/controllermanager/main.go
+++ b/cmd/controllermanager/main.go
@@ -7,17 +7,14 @@ import (
"io/ioutil"
"os"
- // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
- // to ensure that exec-entrypoint and run can make use of them.
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"gopkg.in/yaml.v2"
- "k8s.io/client-go/kubernetes"
- _ "k8s.io/client-go/plugin/pkg/client/auth"
-
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+ "k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+ _ "k8s.io/client-go/plugin/pkg/client/auth"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -26,7 +23,6 @@ import (
"github.com/substratusai/substratus/internal/cloud"
"github.com/substratusai/substratus/internal/controller"
"github.com/substratusai/substratus/internal/sci"
- //+kubebuilder:scaffold:imports
)
var (
diff --git a/cmd/sub/main.go b/cmd/sub/main.go
new file mode 100644
index 00000000..cd45ab92
--- /dev/null
+++ b/cmd/sub/main.go
@@ -0,0 +1,16 @@
+package main
+
+import (
+ "log"
+
+ "github.com/substratusai/substratus/internal/cli"
+)
+
+var Version = "development"
+
+func main() {
+ cli.Version = Version
+ if err := cli.Command().Execute(); err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/config/crd/bases/substratus.ai_models.yaml b/config/crd/bases/substratus.ai_models.yaml
index fe03196d..4fe6f94d 100644
--- a/config/crd/bases/substratus.ai_models.yaml
+++ b/config/crd/bases/substratus.ai_models.yaml
@@ -44,16 +44,6 @@ spec:
spec:
description: Spec is the desired state of the Model.
properties:
- baseModel:
- description: BaseModel should be set in order to mount another model
- to be used for transfer learning.
- properties:
- name:
- description: Name of Kubernetes object.
- type: string
- required:
- - name
- type: object
build:
description: Build specifies how to build an image.
properties:
@@ -113,6 +103,15 @@ spec:
items:
type: string
type: array
+ dataset:
+ description: Dataset to mount for training.
+ properties:
+ name:
+ description: Name of Kubernetes object.
+ type: string
+ required:
+ - name
+ type: object
env:
additionalProperties:
type: string
@@ -121,6 +120,16 @@ spec:
image:
description: Image that contains model code and dependencies.
type: string
+ model:
+ description: Model should be set in order to mount another model to
+ be used for transfer learning.
+ properties:
+ name:
+ description: Name of Kubernetes object.
+ type: string
+ required:
+ - name
+ type: object
params:
additionalProperties:
anyOf:
@@ -161,15 +170,6 @@ spec:
format: int64
type: integer
type: object
- trainingDataset:
- description: Dataset to mount for training.
- properties:
- name:
- description: Name of Kubernetes object.
- type: string
- required:
- - name
- type: object
type: object
status:
description: Status is the observed state of the Model.
diff --git a/config/crd/bases/substratus.ai_notebooks.yaml b/config/crd/bases/substratus.ai_notebooks.yaml
index afa26e35..af679812 100644
--- a/config/crd/bases/substratus.ai_notebooks.yaml
+++ b/config/crd/bases/substratus.ai_notebooks.yaml
@@ -180,6 +180,12 @@ spec:
status:
description: Status is the observed state of the Notebook.
properties:
+ artifacts:
+ description: Artifacts status.
+ properties:
+ url:
+ type: string
+ type: object
buildUpload:
description: BuildUpload contains the status of the build context
upload.
diff --git a/containertools/cmd/nbwatch/main.go b/containertools/cmd/nbwatch/main.go
index ed06530e..baa2df02 100644
--- a/containertools/cmd/nbwatch/main.go
+++ b/containertools/cmd/nbwatch/main.go
@@ -8,14 +8,15 @@ import (
"path/filepath"
"strings"
- "k8s.io/klog/v2"
-
"github.com/fsnotify/fsnotify"
)
var Version = "development"
func main() {
+ log.SetOutput(os.Stderr)
+ log.Println("Starting")
+
if len(os.Args) == 2 && os.Args[1] == "version" {
fmt.Printf("nbwatch %s\n", Version)
os.Exit(0)
@@ -33,7 +34,31 @@ func run() error {
}
defer w.Close()
- w.Add("/content/src")
+ const contentDir = "/content"
+
+ // NOTE: Watch is non-recursive.
+ log.Printf("Watching: %v", contentDir)
+ w.Add(contentDir)
+
+ entries, err := os.ReadDir(contentDir)
+ if err != nil {
+ return fmt.Errorf("reading dir: %w", err)
+ }
+ for _, e := range entries {
+ if !e.IsDir() {
+ continue
+ }
+
+ switch name := e.Name(); name {
+ case "data", "model", "artifacts":
+ default:
+ if !strings.HasPrefix(name, ".") {
+ p := filepath.Join(contentDir, name)
+ log.Printf("Watching: %v", p)
+ w.Add(p)
+ }
+ }
+ }
watchLoop(w)
@@ -49,7 +74,7 @@ func watchLoop(w *fsnotify.Watcher) {
if !ok { // Channel was closed (i.e. Watcher.Close() was called).
return
}
- klog.Error(err)
+ log.Printf("error: %v", err)
// Read from Events.
case e, ok := <-w.Events:
if !ok { // Channel was closed (i.e. Watcher.Close() was called).
@@ -59,9 +84,10 @@ func watchLoop(w *fsnotify.Watcher) {
i++
path := e.Name
+ base := filepath.Base(path)
// Covers ".git", ".gitignore", ".gitmodules", ".gitattributes", ".ipynb_checkpoints"
// and also temporary files that Jupyter writes on save like: ".~hello.py"
- if strings.HasPrefix(filepath.Base(path), ".") {
+ if strings.HasPrefix(base, ".") {
continue
}
diff --git a/docs/cli.md b/docs/cli.md
new file mode 100644
index 00000000..202d6dac
--- /dev/null
+++ b/docs/cli.md
@@ -0,0 +1,316 @@
+# CLI
+
+Operates on directories as a unit of work:
+
+```
+my-model/
+ substratus.yaml <---- Reads Dataset/Model/Server/Notebook from here.
+ Dockerfile <--- Optional
+ run.ipynb
+```
+
+## Alternative Names
+
+### "ai" CLI
+
+Pros
+
+* Short and sweet
+
+Cons
+
+* Generic
+
+```bash
+# Looks for "ai.yaml"...
+
+ai notebook .
+ai nb .
+ai run .
+```
+
+### "strat" CLI
+
+Pros
+
+* More specific than `sub`
+
+Cons
+
+* Longer than `sub`
+
+```bash
+strat notebook .
+strat nb .
+strat run .
+```
+
+## Notebook
+
+```bash
+sub notebook .
+sub nb .
+```
+
+## Get
+
+```bash
+sub get
+```
+
+```
+sub get
+
+models/
+ facebook-opt-125m
+ falcon-7b
+
+datasets/
+ squad
+
+servers/
+ falcon-7b
+```
+
+```
+sub get models
+
+facebook-opt-125m
+falcon-7b
+```
+
+```
+sub get models/falcon-7b
+
+v3
+v2
+v1
+```
+
+```
+sub get models/falcon-7b.v3
+
+metrics:
+ loss: 0.9
+ abc: 123
+```
+
+## Apply
+
+```
+# Alternative names???
+
+# "run" --> currently prefer "apply" b/c the target is a noun/end-object (Model / Dataset / Server)
+sub run .
+```
+
+### Apply (with `
` arg)
+
+* Tar & upload
+* Remote build
+* Wait for Ready
+
+```bash
+sub apply .
+```
+
+## View
+
+* Grab `run.html` (converted notebook) and serve on localhost.
+* Open browser.
+
+```bash
+sub view model/falcon-7b
+sub view dataset/squad
+```
+
+Alternative names:
+
+```bash
+sub logs
+sub show
+sub inspect
+```
+
+## Delete
+
+```bash
+sub delete /
+sub del
+
+# By name
+sub delete models/facebook-opt-125m
+sub delete datasets/squad
+```
+
+## Inference Client
+
+```bash
+sub infer
+sub inf
+
+# OR:
+sub cl (client)
+sub ch (chat) # LLM prompt/completion
+sub cl (classify) # Image recognition
+```
+
+https://github.com/charmbracelet/bubbletea/tree/master/examples#chat
+
+
+## Substratus.yaml
+
+### Option 1: Workspace file
+
+A "workspace" file could represent multiple different objects.
+
+```yaml
+apiVersion: substratus.ai/v1
+metadata:
+ name: snakes-opt-125m
+model:
+ dataset:
+ name: snakes
+ model:
+ name: facebook-opt-125m
+dataset:
+server:
+ # Generated if not specified
+ # Only valid if model is specified.
+notebook:
+ # Generated if not specified
+```
+
+### Option 2: Multi-doc
+
+**Pros:**
+
+* No new objects
+
+**Cons:**
+
+* Duplication of fields like `.metadata`
+
+```yaml
+apiVersion: substratus.ai/v1
+kind: Model
+metadata:
+ name: snakes-opt-125m
+spec:
+ dataset:
+ name: snakes
+ model:
+ name: facebook-opt-125m
+---
+apiVersion: substratus.ai/v1
+kind: Server
+metadata:
+ name: snakes-opt-125m
+spec:
+ # ...
+---
+apiVersion: substratus.ai/v1
+kind: Notebook
+metadata:
+ name: snakes-opt-125m
+spec:
+ # ...
+```
+
+### Option 3: Directory
+
+**Pros:**
+
+* No new objects
+* Options for more than 1 objects
+
+**Cons:**
+
+* Duplication of fields like `.metadata`
+* Messy
+
+```
+.substratus/
+ dataset.yaml
+ model.yaml
+ notebook.yaml
+ server.yaml
+
+my-code.ipynb
+```
+
+```yaml
+apiVersion: substratus.ai/v1
+kind: Model
+metadata:
+ name: snakes-opt-125m
+spec:
+ dataset:
+ name: snakes
+ model:
+ name: facebook-opt-125m
+---
+apiVersion: substratus.ai/v1
+kind: Server
+metadata:
+ name: snakes-opt-125m
+spec:
+ # ...
+---
+apiVersion: substratus.ai/v1
+kind: Notebook
+metadata:
+ name: snakes-opt-125m
+spec:
+ # ...
+```
+
+### Option 4: Kustomize-like
+
+**Pros:**
+
+* No new objects
+* Options for more than 1 objects
+* Ability to express remote base-model/dataset dependencies
+
+**Cons:**
+
+* Could get messy
+
+```
+substratus.yaml
+
+dataset.yaml
+model.yaml
+notebook.yaml
+server.yaml
+
+my-code.ipynb
+```
+
+```yaml
+Model: model.yaml
+dependencies:
+- file: ./model.yaml
+- https: //raw.githubusercontent.com/substratusai/substratus/main/install/kind/manifests.yaml
+- gcs: /some-bucket/some/file.yaml
+```
+
+### Option 5: Multi-doc with remotes
+
+```yaml
+apiVersion: substratus.ai/v1
+kind: Model
+metadata:
+ name: snakes-opt-125m
+spec:
+ dataset:
+ name: snakes
+ model:
+ name: facebook-opt-125m
+---
+- ../base-model.yaml
+- https://raw.githubusercontent.com/substratusai/substratus/main/examples/facebook-opt-125m.yaml
+- https://raw.githubusercontent.com/substratusai/substratus/main/examples/squad-dataset.yaml
+```
+
diff --git a/docs/container-contract.md b/docs/container-contract.md
index d643f7a4..a947ffff 100644
--- a/docs/container-contract.md
+++ b/docs/container-contract.md
@@ -25,12 +25,10 @@ Note: This requirement is satisfied by default when using Substratus base images
## Directory Structure
```
-/content/ # Working directory.
- data/ # Location where Datasets will be mounted (reading and loading).
- src/ # Source code (*.py, *.ipynb) for loading, training, etc.
- logs/ # Output of building/training jobs for debugging purposes.
- model/ # Location to store the resulting model from loading or training.
- saved-model/ # Location where a previously saved model will be mounted.
+/content/ # Working directory.
+ data/ # Location where a previously stored Datasets is mounted.
+ model/ # Location where a previously stored Model is mounted.
+ artifacts/ # Location to store output of a run.
```
## Parameters
diff --git a/docs/design.md b/docs/design.md
index 0dcd0567..8e4b68c4 100644
--- a/docs/design.md
+++ b/docs/design.md
@@ -113,11 +113,9 @@ The following scheme can be used for storing artifacts for Models, Datasets, and
```sh
# Models
gs://{bucket}/{hash}/model # Model artifacts (*.pt, etc)
-gs://{bucket}/{hash}/logs # Logs and converted notebooks
# Datasets
gs://{bucket}/{hash}/data # Data artifacts (*.jsonl, etc)
-gs://{bucket}/{hash}/logs # Logs and converted notebooks
# Notebooks
gs://{bucket}/{hash}/build/{md5-checksum}.tar # Uploaded build context
@@ -127,7 +125,6 @@ The example Model's backing storage would end up being:
```sh
gs://abc123-substratus-artifacts/f94a0d128bcbd9c1b824e9e5572baf86/model/
-gs://abc123-substratus-artifacts/f94a0d128bcbd9c1b824e9e5572baf86/logs/
```
The Model would report this URL in its status:
@@ -166,15 +163,12 @@ runJob()
All mount points will are made under a standardized `/content` directory which should correspond to the `WORKDIR` of a Dockerfile. This works well for Jupyter notebooks which can be run with `/content` set as the serving directory: all relevant mounts will be populated on the file explorer sidebar.
-The `logs/` directories below are used to store the notebook cell output, python logs, tensorboard stats, etc. These directories are mounted as read-only in Notebooks to explore the output of other background jobs.
-
##### Dataset (importing)
```
/content/params.json # Populated from .spec.params (also available as env vars).
-/content/data/ # Mounted RW: initially empty dir to write new files
-/content/logs/ # Mounted RW: initially empty dir to write new files
+/content/artifacts/ # Mounted RW: initially empty dir to write new files
```
##### Model (importing)
@@ -182,8 +176,7 @@ The `logs/` directories below are used to store the notebook cell output, python
```
/content/params.json # Populated from .spec.params (also available as env vars).
-/content/model/ # Mounted RW: initially empty dir to write new files
-/content/logs/ # Mounted RW: initially empty dir to write new files
+/content/artifacts/ # Mounted RW: initially empty dir to write new files
```
##### Model (training)
@@ -191,12 +184,11 @@ The `logs/` directories below are used to store the notebook cell output, python
```
/content/params.json # Populated from .spec.params (also available as env vars).
-/content/data/ # Mounted RO: from .spec.trainingDataset
+/content/data/ # Mounted RO: from .spec.dataset
-/content/saved-model/ # Mounted RO: from .spec.baseModel
+/content/model/ # Mounted RO: from .spec.model
-/content/model/ # Mounted RW: initially empty dir for writing new files
-/content/logs/ # Mounted RW: initially empty dir for writing logs
+/content/artifacts/ # Mounted RW: initially empty dir for writing new files
```
##### Notebook
@@ -207,10 +199,10 @@ NOTE: The `saved-model/` directory is the same as the container for the Model ob
/content/params.json # Populated from .spec.params (also available as env vars).
/content/data/ # Mounted RO: from .spec.dataset
-/content/data-logs/ # Mounted RO: from .spec.dataset
-/content/saved-model/ # Mounted RO: from .spec.model
-/content/saved-model-logs/ # Mounted RO: from .spec.model
+/content/model/ # Mounted RO: from .spec.model
+
+/content/artifacts/ # Mounted RW: initially empty dir for writing new files
```
##### Server:
@@ -218,7 +210,7 @@ NOTE: The `saved-model/` directory is the same as the container for the Model ob
```
/content/params.json # Populated from .spec.params (also available as env vars).
-/content/saved-model/ # Mounted RO: from .spec.model
+/content/model/ # Mounted RO: from .spec.model
```
### Naming
diff --git a/docs/development.md b/docs/development.md
index 41ae51bc..c5253680 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -38,19 +38,35 @@ go build ./kubectl/cmd/applybuild &&
The `kubectl notebook` command depends on container-tools for live-syncing. The plugin will try
to download these tools from GitHub releases if they dont already exist with the right versions.
-You can build the container-tools for development purposes using the following. NOTE: This is the default cache directory on a mac, this will be different on other machine types.
+You can build the container-tools for development purposes using the following.
+Mac steps:
```sh
export NODE_ARCH=amd64
rm -rf /Users/$USER/Library/Caches/substratus
mkdir -p /Users/$USER/Library/Caches/substratus/container-tools/$NODE_ARCH
GOOS=linux GOARCH=$NODE_ARCH go build ./containertools/cmd/nbwatch
+
mv nbwatch /Users/$USER/Library/Caches/substratus/container-tools/$NODE_ARCH/
echo "development" > /Users/$USER/Library/Caches/substratus/container-tools/version.txt
```
+Linux steps:
+```sh
+export NODE_ARCH=amd64
+
+rm -rf ~/.cache/substratus
+mkdir -p ~/.cache/substratus/container-tools/$NODE_ARCH
+GOOS=linux GOARCH=$NODE_ARCH go build ./containertools/cmd/nbwatch
+
+mv nbwatch ~/.cache/substratus/container-tools/amd64/
+
+echo "development" > ~/.cache/substratus/container-tools/version.txt
+```
+
+
### Install from release
Release binaries are created for most architectures when the repo is tagged.
diff --git a/examples/facebook-opt-125m/finetuned-model-gitops.yaml b/examples/facebook-opt-125m/finetuned-model-gitops.yaml
index ec030699..50b79e17 100644
--- a/examples/facebook-opt-125m/finetuned-model-gitops.yaml
+++ b/examples/facebook-opt-125m/finetuned-model-gitops.yaml
@@ -7,9 +7,9 @@ spec:
git:
url: https://github.com/substratusai/images
path: model-trainer-huggingface
- baseModel:
+ model:
name: facebook-opt-125m
- trainingDataset:
+ dataset:
name: squad
params:
epochs: 1
diff --git a/examples/facebook-opt-125m/finetuned-model.yaml b/examples/facebook-opt-125m/finetuned-model.yaml
index 78c89262..8b102005 100644
--- a/examples/facebook-opt-125m/finetuned-model.yaml
+++ b/examples/facebook-opt-125m/finetuned-model.yaml
@@ -4,9 +4,9 @@ metadata:
name: fb-opt-125m-squad
spec:
image: substratusai/model-trainer-huggingface
- baseModel:
+ model:
name: facebook-opt-125m
- trainingDataset:
+ dataset:
name: squad
params:
epochs: 1
diff --git a/examples/falcon-40b/finetuned-model.yaml b/examples/falcon-40b/finetuned-model.yaml
index 2ae3fc3b..8928468d 100644
--- a/examples/falcon-40b/finetuned-model.yaml
+++ b/examples/falcon-40b/finetuned-model.yaml
@@ -4,9 +4,9 @@ metadata:
name: falcon-40b-k8s
spec:
image: substratusai/model-trainer-huggingface
- baseModel:
+ model:
name: falcon-40b
- trainingDataset:
+ dataset:
name: k8s-instructions
params:
epochs: 1
diff --git a/examples/falcon-7b-instruct/finetuned-model-custom-prompt.yaml b/examples/falcon-7b-instruct/finetuned-model-custom-prompt.yaml
index 21e77f7d..fd78fc97 100644
--- a/examples/falcon-7b-instruct/finetuned-model-custom-prompt.yaml
+++ b/examples/falcon-7b-instruct/finetuned-model-custom-prompt.yaml
@@ -4,9 +4,9 @@ metadata:
name: falcon-7b-instruct-k8s-custom-prompt
spec:
image: substratusai/model-trainer-huggingface
- baseModel:
+ model:
name: falcon-7b-instruct
- trainingDataset:
+ dataset:
name: k8s-instructions
params:
num_train_epochs: 1
diff --git a/examples/falcon-7b-instruct/finetuned-model.yaml b/examples/falcon-7b-instruct/finetuned-model.yaml
index a63113ec..60220136 100644
--- a/examples/falcon-7b-instruct/finetuned-model.yaml
+++ b/examples/falcon-7b-instruct/finetuned-model.yaml
@@ -4,9 +4,9 @@ metadata:
name: falcon-7b-instruct-k8s
spec:
image: substratusai/model-trainer-huggingface
- baseModel:
+ model:
name: falcon-7b-instruct
- trainingDataset:
+ dataset:
name: k8s-instructions
params:
# See HuggingFace transformers.TrainingArguments for all parameters
diff --git a/examples/llama2-7b/finetuned-model.yaml b/examples/llama2-7b/finetuned-model.yaml
index 7327f428..a4339cb4 100644
--- a/examples/llama2-7b/finetuned-model.yaml
+++ b/examples/llama2-7b/finetuned-model.yaml
@@ -4,9 +4,9 @@ metadata:
name: llama-2-7b-k8s
spec:
image: substratusai/model-trainer-huggingface
- baseModel:
+ model:
name: llama-2-7b
- trainingDataset:
+ dataset:
name: k8s-instructions
params:
# See HuggingFace transformers.TrainingArguments for all parameters
diff --git a/examples/notebook/src/hello.py b/examples/notebook/src/hello.py
index 8c35c8ee..4039db7a 100644
--- a/examples/notebook/src/hello.py
+++ b/examples/notebook/src/hello.py
@@ -1 +1 @@
-print("hello!!!")
+print("hello!!!")
\ No newline at end of file
diff --git a/go.mod b/go.mod
index 84656a75..8b64ba65 100644
--- a/go.mod
+++ b/go.mod
@@ -1,17 +1,21 @@
module github.com/substratusai/substratus
-go 1.19
+go 1.21
require (
cloud.google.com/go/compute/metadata v0.2.3
cloud.google.com/go/storage v1.31.0
- github.com/briandowns/spinner v1.23.0
+ github.com/aws/aws-sdk-go v1.44.321
+ github.com/charmbracelet/bubbles v0.16.1
+ github.com/charmbracelet/bubbletea v0.24.2
+ github.com/charmbracelet/lipgloss v0.8.0
github.com/go-logr/logr v1.2.4
github.com/go-playground/validator/v10 v10.14.1
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
github.com/sethvargo/go-envconfig v0.9.0
github.com/spf13/cobra v1.6.0
github.com/stretchr/testify v1.8.4
+ golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb
google.golang.org/grpc v1.57.0
google.golang.org/protobuf v1.31.0
k8s.io/api v0.27.4
@@ -25,9 +29,12 @@ require (
require github.com/inconshreveable/mousetrap v1.0.1 // indirect
require (
- github.com/aws/aws-sdk-go v1.44.321 // indirect
cloud.google.com/go v0.110.6 // indirect
cloud.google.com/go/compute v1.23.0 // indirect
+ github.com/atotto/clipboard v0.1.4 // indirect
+ github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+ github.com/charmbracelet/harmonica v0.2.0 // indirect
+ github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/go-errors/errors v1.4.2 // indirect
@@ -35,15 +42,22 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
- github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
+ github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
+ github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+ github.com/mattn/go-localereader v0.0.1 // indirect
+ github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+ github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect
+ github.com/muesli/cancelreader v0.2.2 // indirect
+ github.com/muesli/reflow v0.3.0 // indirect
+ github.com/muesli/termenv v0.15.2 // indirect
+ github.com/rivo/uniseg v0.2.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/xlab/treeprint v1.1.0 // indirect
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
golang.org/x/crypto v0.12.0 // indirect
- golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect
@@ -59,7 +73,6 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
- github.com/fatih/color v1.7.0 // indirect
github.com/fsnotify/fsnotify v1.6.0
github.com/go-logr/zapr v1.2.4 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
@@ -77,8 +90,7 @@ require (
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
- github.com/mattn/go-colorable v0.1.2 // indirect
- github.com/mattn/go-isatty v0.0.8 // indirect
+ github.com/mattn/go-isatty v0.0.18 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
diff --git a/go.sum b/go.sum
index 36ad428e..a1da0bf8 100644
--- a/go.sum
+++ b/go.sum
@@ -13,18 +13,29 @@ cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDT
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
+github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
github.com/aws/aws-sdk-go v1.44.321 h1:iXwFLxWjZPjYqjPq0EcCs46xX7oDLEELte1+BzgpKk8=
github.com/aws/aws-sdk-go v1.44.321/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
+github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A=
-github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY=
+github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc=
+github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY=
+github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg=
+github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ=
+github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao=
+github.com/charmbracelet/lipgloss v0.8.0 h1:IS00fk4XAHcf8uZKc3eHeMUTCxUH6NkaTrdyCQk84RU=
+github.com/charmbracelet/lipgloss v0.8.0/go.mod h1:p4eYUZZJ/0oXTuCQKFF8mqyKCz0ja6y+7DniDDw5KKU=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -35,6 +46,8 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY=
+github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -53,8 +66,6 @@ github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH
github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
-github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
@@ -62,10 +73,6 @@ github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -78,6 +85,7 @@ github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En
github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
@@ -85,6 +93,7 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91
github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k=
github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -123,7 +132,9 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
+github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
+github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
@@ -144,34 +155,36 @@ github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
+github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
+github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
+github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
+github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
+github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
+github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
@@ -183,11 +196,20 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
+github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b h1:1XF24mVaiu7u+CFywTdcDo2ie1pzzhwjt6RHqzpMU34=
+github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho=
+github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
+github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
+github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
+github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
+github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo=
+github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
+github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU=
+github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -204,10 +226,15 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI
github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/sethvargo/go-envconfig v0.9.0 h1:Q6FQ6hVEeTECULvkJZakq3dZMeBQ3JUpcKMfPQbKMDE=
github.com/sethvargo/go-envconfig v0.9.0/go.mod h1:Iz1Gy1Sf3T64TQlJSvee81qDhf7YIlt8GMUX6yyNFs0=
github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI=
@@ -243,6 +270,7 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
+go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
@@ -265,8 +293,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
-golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -283,6 +309,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -301,7 +328,6 @@ golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -315,10 +341,13 @@ golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -327,6 +356,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
@@ -342,6 +372,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
+golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/internal/cli/common.go b/internal/cli/common.go
new file mode 100644
index 00000000..9bb8628d
--- /dev/null
+++ b/internal/cli/common.go
@@ -0,0 +1,15 @@
+package cli
+
+import (
+ "k8s.io/client-go/kubernetes/scheme"
+
+ apiv1 "github.com/substratusai/substratus/api/v1"
+ "github.com/substratusai/substratus/internal/client"
+)
+
+func init() {
+ apiv1.AddToScheme(scheme.Scheme)
+}
+
+// NewClient is a dirty hack to allow the client to be mocked out in tests.
+var NewClient = client.NewClient
diff --git a/internal/cli/delete.go b/internal/cli/delete.go
new file mode 100644
index 00000000..b576ed6c
--- /dev/null
+++ b/internal/cli/delete.go
@@ -0,0 +1,78 @@
+package cli
+
+import (
+ "fmt"
+ "os"
+
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/spf13/cobra"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+
+ "github.com/substratusai/substratus/internal/cli/utils"
+ "github.com/substratusai/substratus/internal/tui"
+)
+
+func deleteCommand() *cobra.Command {
+ var flags struct {
+ namespace string
+ filename string
+ kubeconfig string
+ }
+
+ run := func(cmd *cobra.Command, args []string) error {
+ defer tui.LogFile.Close()
+
+ kubeconfigNamespace, restConfig, err := utils.BuildConfigFromFlags("", flags.kubeconfig)
+ if err != nil {
+ return fmt.Errorf("rest config: %w", err)
+ }
+
+ clientset, err := kubernetes.NewForConfig(restConfig)
+ if err != nil {
+ return fmt.Errorf("clientset: %w", err)
+ }
+
+ c := NewClient(clientset, restConfig)
+
+ // Initialize our program
+ tui.P = tea.NewProgram((&tui.DeleteModel{
+ Ctx: cmd.Context(),
+ Scope: args[0],
+ Namespace: tui.Namespace{
+ Contextual: kubeconfigNamespace,
+ Specified: flags.namespace,
+ },
+ Client: c,
+ }).New())
+ if _, err := tui.P.Run(); err != nil {
+ return err
+ }
+
+ return nil
+ }
+
+ cmd := &cobra.Command{
+ Use: "delete",
+ Aliases: []string{"del"},
+ Short: "Delete a Substratus Dataset, Model, Server, or Notebook",
+ Args: cobra.MaximumNArgs(1),
+ Run: func(cmd *cobra.Command, args []string) {
+ if err := run(cmd, args); err != nil {
+ fmt.Fprintln(os.Stderr, err)
+ os.Exit(1)
+ }
+ },
+ }
+
+ defaultKubeconfig := os.Getenv("KUBECONFIG")
+ if defaultKubeconfig == "" {
+ defaultKubeconfig = clientcmd.RecommendedHomeFile
+ }
+ cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "")
+
+ cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook")
+ cmd.Flags().StringVarP(&flags.filename, "filename", "f", "", "Manifest file")
+
+ return cmd
+}
diff --git a/internal/cli/get.go b/internal/cli/get.go
new file mode 100644
index 00000000..0db3387c
--- /dev/null
+++ b/internal/cli/get.go
@@ -0,0 +1,85 @@
+package cli
+
+import (
+ "fmt"
+ "os"
+
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/spf13/cobra"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+
+ "github.com/substratusai/substratus/internal/cli/utils"
+ "github.com/substratusai/substratus/internal/tui"
+)
+
+func getCommand() *cobra.Command {
+ var flags struct {
+ namespace string
+ kubeconfig string
+ }
+
+ run := func(cmd *cobra.Command, args []string) error {
+ defer tui.LogFile.Close()
+
+ kubeconfigNamespace, restConfig, err := utils.BuildConfigFromFlags("", flags.kubeconfig)
+ if err != nil {
+ return fmt.Errorf("rest config: %w", err)
+ }
+
+ namespace := "default"
+ if flags.namespace != "" {
+ namespace = flags.namespace
+ } else if kubeconfigNamespace != "" {
+ namespace = kubeconfigNamespace
+ }
+
+ clientset, err := kubernetes.NewForConfig(restConfig)
+ if err != nil {
+ return fmt.Errorf("clientset: %w", err)
+ }
+
+ c := NewClient(clientset, restConfig)
+
+ var scope string
+ if len(args) > 0 {
+ scope = args[0]
+ }
+
+ // Initialize our program
+ tui.P = tea.NewProgram((&tui.GetModel{
+ Ctx: cmd.Context(),
+ Scope: scope,
+ Namespace: namespace,
+
+ Client: c,
+ }).New() /*, tea.WithAltScreen()*/)
+ if _, err := tui.P.Run(); err != nil {
+ return err
+ }
+
+ return nil
+ }
+
+ cmd := &cobra.Command{
+ Use: "get",
+ Short: "Get Substratus Datasets, Models, Notebooks, and Servers",
+ Args: cobra.MaximumNArgs(1),
+ Run: func(cmd *cobra.Command, args []string) {
+ if err := run(cmd, args); err != nil {
+ fmt.Fprintln(os.Stderr, err)
+ os.Exit(1)
+ }
+ },
+ }
+
+ defaultKubeconfig := os.Getenv("KUBECONFIG")
+ if defaultKubeconfig == "" {
+ defaultKubeconfig = clientcmd.RecommendedHomeFile
+ }
+ cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "")
+
+ cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook")
+
+ return cmd
+}
diff --git a/internal/cli/infer.go b/internal/cli/infer.go
new file mode 100644
index 00000000..b6b30df6
--- /dev/null
+++ b/internal/cli/infer.go
@@ -0,0 +1,82 @@
+package cli
+
+import (
+ "fmt"
+ "os"
+
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/spf13/cobra"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+
+ "github.com/substratusai/substratus/internal/cli/utils"
+ "github.com/substratusai/substratus/internal/tui"
+)
+
+func inferCommand() *cobra.Command {
+ var flags struct {
+ namespace string
+ kubeconfig string
+ }
+
+ run := func(cmd *cobra.Command, args []string) error {
+ defer tui.LogFile.Close()
+
+ kubeconfigNamespace, restConfig, err := utils.BuildConfigFromFlags("", flags.kubeconfig)
+ if err != nil {
+ return fmt.Errorf("rest config: %w", err)
+ }
+
+ namespace := "default"
+ if flags.namespace != "" {
+ namespace = flags.namespace
+ } else if kubeconfigNamespace != "" {
+ namespace = kubeconfigNamespace
+ }
+
+ clientset, err := kubernetes.NewForConfig(restConfig)
+ if err != nil {
+ return fmt.Errorf("clientset: %w", err)
+ }
+
+ _ = namespace
+
+ c := NewClient(clientset, restConfig)
+ _ = c
+
+ // Initialize our program
+ // TODO: Use a differnt tui-model for different types of Model objects:
+ // ex: Vector-search TUI, Image recongnition TUI
+ m := tui.ChatModel{}
+
+ tui.P = tea.NewProgram(m)
+ if _, err := tui.P.Run(); err != nil {
+ return err
+ }
+
+ return nil
+ }
+
+ cmd := &cobra.Command{
+ Use: "infer",
+ Aliases: []string{"in"},
+ Short: "Run inference against a Substratus Server",
+ Args: cobra.MaximumNArgs(1),
+ Run: func(cmd *cobra.Command, args []string) {
+ if err := run(cmd, args); err != nil {
+ fmt.Fprintln(os.Stderr, err)
+ os.Exit(1)
+ }
+ },
+ }
+
+ defaultKubeconfig := os.Getenv("KUBECONFIG")
+ if defaultKubeconfig == "" {
+ defaultKubeconfig = clientcmd.RecommendedHomeFile
+ }
+ cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "")
+
+ cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook")
+
+ return cmd
+}
diff --git a/internal/cli/notebook.go b/internal/cli/notebook.go
new file mode 100644
index 00000000..a8abf554
--- /dev/null
+++ b/internal/cli/notebook.go
@@ -0,0 +1,158 @@
+package cli
+
+import (
+ "fmt"
+ "os"
+
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/spf13/cobra"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+
+ "github.com/substratusai/substratus/internal/cli/utils"
+ "github.com/substratusai/substratus/internal/tui"
+)
+
+func notebookCommand() *cobra.Command {
+ var flags struct {
+ resume string
+ namespace string
+ filename string
+ kubeconfig string
+ fullscreen bool
+ }
+
+ run := func(cmd *cobra.Command, args []string) error {
+ defer tui.LogFile.Close()
+
+ //if flags.filename == "" {
+ // defaultFilename := "notebook.yaml"
+ // if _, err := os.Stat(filepath.Join(args[0], "notebook.yaml")); err == nil {
+ // flags.filename = defaultFilename
+ // } else {
+ // return fmt.Errorf("Flag -f (--filename) required when default notebook.yaml file does not exist")
+ // }
+ //}
+
+ kubeconfigNamespace, restConfig, err := utils.BuildConfigFromFlags("", flags.kubeconfig)
+ if err != nil {
+ return fmt.Errorf("rest config: %w", err)
+ }
+
+ //namespace := "default"
+ //if flags.namespace != "" {
+ // namespace = flags.namespace
+ //} else if kubeconfigNamespace != "" {
+ // namespace = kubeconfigNamespace
+ //}
+
+ clientset, err := kubernetes.NewForConfig(restConfig)
+ if err != nil {
+ return fmt.Errorf("clientset: %w", err)
+ }
+
+ c := NewClient(clientset, restConfig)
+ //notebooks, err := c.Resource(&apiv1.Notebook{
+ // TypeMeta: metav1.TypeMeta{
+ // APIVersion: "substratus.ai/v1",
+ // Kind: "Notebook",
+ // },
+ //})
+ //if err != nil {
+ // return fmt.Errorf("resource client: %w", err)
+ //}
+
+ //var obj client.Object
+ //if flags.resume != "" {
+ // fetched, err := notebooks.Get(namespace, flags.resume)
+ // if err != nil {
+ // return fmt.Errorf("getting notebook: %w", err)
+ // }
+ // obj = fetched.(client.Object)
+ //} else {
+ // manifest, err := os.ReadFile(flags.filename)
+ // if err != nil {
+ // return fmt.Errorf("reading file: %w", err)
+ // }
+ // obj, err = client.Decode(manifest)
+ // if err != nil {
+ // return fmt.Errorf("decoding: %w", err)
+ // }
+ // if obj.GetNamespace() == "" {
+ // // When there is no .metadata.namespace set in the manifest...
+ // obj.SetNamespace(namespace)
+ // } else {
+ // // TODO: Closer match kubectl behavior here by differentiaing between
+ // // the short -n and long --namespace flags.
+ // // See example kubectl error:
+ // // error: the namespace from the provided object "a" does not match the namespace "b". You must pass '--namespace=a' to perform this operation.
+ // if flags.namespace != "" && flags.namespace != obj.GetNamespace() {
+ // // When there is .metadata.namespace set in the manifest and
+ // // a conflicting -n or --namespace flag...
+ // return fmt.Errorf("the namespace from the provided object %q does not match the namespace %q from flag", obj.GetNamespace(), flags.namespace)
+ // }
+ // }
+ //}
+
+ //nb, err := client.NotebookForObject(obj)
+ //if err != nil {
+ // return fmt.Errorf("notebook for object: %w", err)
+ //}
+ //nb.Spec.Suspend = ptr.To(false)
+
+ var pOpts []tea.ProgramOption
+ if flags.fullscreen {
+ pOpts = append(pOpts, tea.WithAltScreen())
+ }
+
+ path := "."
+ if len(args) > 0 {
+ path = args[0]
+ }
+
+ // Initialize our program
+ tui.P = tea.NewProgram((&tui.NotebookModel{
+ Ctx: cmd.Context(),
+ Path: path,
+ Filename: flags.filename,
+ Namespace: tui.Namespace{
+ Contextual: kubeconfigNamespace,
+ Specified: flags.namespace,
+ },
+ Client: c,
+ K8s: clientset,
+ }).New(), pOpts...)
+ if _, err := tui.P.Run(); err != nil {
+ return err
+ }
+
+ return nil
+ }
+
+ cmd := &cobra.Command{
+ Use: "notebook [dir]",
+ Aliases: []string{"nb"},
+ Short: "Start a Jupyter Notebook development environment",
+ Args: cobra.MaximumNArgs(1),
+ Run: func(cmd *cobra.Command, args []string) {
+ if err := run(cmd, args); err != nil {
+ fmt.Fprintln(os.Stderr, err)
+ os.Exit(1)
+ }
+ },
+ }
+
+ defaultKubeconfig := os.Getenv("KUBECONFIG")
+ if defaultKubeconfig == "" {
+ defaultKubeconfig = clientcmd.RecommendedHomeFile
+ }
+ cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "")
+
+ cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook")
+ cmd.Flags().StringVarP(&flags.filename, "filename", "f", "", "Manifest file")
+ cmd.Flags().StringVarP(&flags.resume, "resume", "r", "", "Name of notebook to resume")
+
+ cmd.Flags().BoolVar(&flags.fullscreen, "fullscreen", false, "Fullscreen mode")
+
+ return cmd
+}
diff --git a/internal/cli/root.go b/internal/cli/root.go
new file mode 100644
index 00000000..c4ad1ab5
--- /dev/null
+++ b/internal/cli/root.go
@@ -0,0 +1,23 @@
+package cli
+
+import (
+ "github.com/spf13/cobra"
+)
+
+var Version string
+
+func Command() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "sub",
+ Short: "Substratus CLI",
+ }
+
+ cmd.AddCommand(notebookCommand())
+ cmd.AddCommand(runCommand())
+ cmd.AddCommand(getCommand())
+ // cmd.AddCommand(inferCommand())
+ cmd.AddCommand(deleteCommand())
+ cmd.AddCommand(serveCommand())
+
+ return cmd
+}
diff --git a/internal/cli/run.go b/internal/cli/run.go
new file mode 100644
index 00000000..dbcb0861
--- /dev/null
+++ b/internal/cli/run.go
@@ -0,0 +1,89 @@
+package cli
+
+import (
+ "fmt"
+ "os"
+
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/spf13/cobra"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+
+ "github.com/substratusai/substratus/internal/cli/utils"
+ "github.com/substratusai/substratus/internal/tui"
+)
+
+func runCommand() *cobra.Command {
+ var flags struct {
+ namespace string
+ filename string
+ kubeconfig string
+ }
+
+ run := func(cmd *cobra.Command, args []string) error {
+ defer tui.LogFile.Close()
+
+ kubeconfigNamespace, restConfig, err := utils.BuildConfigFromFlags("", flags.kubeconfig)
+ if err != nil {
+ return fmt.Errorf("rest config: %w", err)
+ }
+
+ clientset, err := kubernetes.NewForConfig(restConfig)
+ if err != nil {
+ return fmt.Errorf("clientset: %w", err)
+ }
+
+ path := "."
+ if len(args) > 0 {
+ path = args[0]
+ }
+
+ tui.P = tea.NewProgram((&tui.RunModel{
+ Ctx: cmd.Context(),
+ Path: path,
+ Filename: flags.filename,
+ Namespace: tui.Namespace{
+ Contextual: kubeconfigNamespace,
+ Specified: flags.namespace,
+ },
+ Client: NewClient(clientset, restConfig),
+ K8s: clientset,
+ }).New())
+ if _, err := tui.P.Run(); err != nil {
+ return err
+ }
+
+ return nil
+ }
+
+ cmd := &cobra.Command{
+ Use: "run [dir]",
+ Short: "Run a local directory. Supported kinds: Dataset, Model.",
+ Example: ` # Upload code from the current directory,
+ # scan *.yaml files looking for Substratus manifests to use.
+ sub run
+
+ # Upload modelling code and create a Model.
+ sub run -f model.yaml .
+
+ # Upoad dataset importing code and create a Dataset.
+ sub run -f dataset.yaml .`,
+ Args: cobra.MaximumNArgs(1),
+ Run: func(cmd *cobra.Command, args []string) {
+ if err := run(cmd, args); err != nil {
+ fmt.Fprintln(os.Stderr, err)
+ os.Exit(1)
+ }
+ },
+ }
+
+ defaultKubeconfig := os.Getenv("KUBECONFIG")
+ if defaultKubeconfig == "" {
+ defaultKubeconfig = clientcmd.RecommendedHomeFile
+ }
+ cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "path to Kubernetes Kubeconfig file")
+ cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "namespace of Notebook")
+ cmd.Flags().StringVarP(&flags.filename, "filename", "f", "", "manifest file")
+
+ return cmd
+}
diff --git a/internal/cli/serve.go b/internal/cli/serve.go
new file mode 100644
index 00000000..8e42b850
--- /dev/null
+++ b/internal/cli/serve.go
@@ -0,0 +1,80 @@
+package cli
+
+import (
+ "fmt"
+ "os"
+
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/spf13/cobra"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+
+ "github.com/substratusai/substratus/internal/cli/utils"
+ "github.com/substratusai/substratus/internal/tui"
+)
+
+func serveCommand() *cobra.Command {
+ var flags struct {
+ namespace string
+ filename string
+ kubeconfig string
+ }
+
+ run := func(cmd *cobra.Command, args []string) error {
+ defer tui.LogFile.Close()
+
+ //if flags.filename == "" {
+ // return fmt.Errorf("Flag -f (--filename) required")
+ //}
+
+ kubeconfigNamespace, restConfig, err := utils.BuildConfigFromFlags("", flags.kubeconfig)
+ if err != nil {
+ return fmt.Errorf("rest config: %w", err)
+ }
+
+ clientset, err := kubernetes.NewForConfig(restConfig)
+ if err != nil {
+ return fmt.Errorf("clientset: %w", err)
+ }
+
+ // Initialize our program
+ tui.P = tea.NewProgram((&tui.ServeModel{
+ Ctx: cmd.Context(),
+ Filename: flags.filename,
+ Namespace: tui.Namespace{
+ Contextual: kubeconfigNamespace,
+ Specified: flags.namespace,
+ },
+ Client: NewClient(clientset, restConfig),
+ K8s: clientset,
+ }).New())
+ if _, err := tui.P.Run(); err != nil {
+ return err
+ }
+
+ return nil
+ }
+
+ cmd := &cobra.Command{
+ Use: "serve",
+ Aliases: []string{"srv"},
+ Short: "Serve a model, open a browser",
+ Run: func(cmd *cobra.Command, args []string) {
+ if err := run(cmd, args); err != nil {
+ fmt.Fprintln(os.Stderr, err)
+ os.Exit(1)
+ }
+ },
+ }
+
+ defaultKubeconfig := os.Getenv("KUBECONFIG")
+ if defaultKubeconfig == "" {
+ defaultKubeconfig = clientcmd.RecommendedHomeFile
+ }
+
+ cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "")
+ cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook")
+ cmd.Flags().StringVarP(&flags.filename, "filename", "f", "", "Manifest file")
+
+ return cmd
+}
diff --git a/kubectl/internal/commands/utils.go b/internal/cli/utils/kubeconfig.go
similarity index 64%
rename from kubectl/internal/commands/utils.go
rename to internal/cli/utils/kubeconfig.go
index 2e993828..9c7f26e9 100644
--- a/kubectl/internal/commands/utils.go
+++ b/internal/cli/utils/kubeconfig.go
@@ -1,34 +1,16 @@
-package commands
+package utils
import (
- "io"
- "os"
-
- "github.com/google/uuid"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"k8s.io/klog/v2"
-
- "github.com/substratusai/substratus/kubectl/internal/client"
)
-var Version = "development"
-
-// NewClient is a dirty hack to allow the client to be mocked out in tests.
-var NewClient = client.NewClient
-
-// NotebookStdout is a dirty hack to allow stdout to be inspected in tests.
-var NotebookStdout io.Writer = os.Stdout
-
-var NewUUID = func() string {
- return uuid.New().String()
-}
-
-// buildConfigFromFlags is a modified version of clientcmd.BuildConfigFromFlags
+// BuildConfigFromFlags is a modified version of clientcmd.BuildConfigFromFlags
// that returns the namespace set in the kubeconfig to make sure we play nicely
// with tools like kubens.
-func buildConfigFromFlags(masterUrl, kubeconfigPath string) (string, *restclient.Config, error) {
+func BuildConfigFromFlags(masterUrl, kubeconfigPath string) (string, *restclient.Config, error) {
if kubeconfigPath == "" && masterUrl == "" {
klog.Warning("Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work.")
kubeconfig, err := restclient.InClusterConfig()
diff --git a/internal/cli/utils/uuid.go b/internal/cli/utils/uuid.go
new file mode 100644
index 00000000..11f6959b
--- /dev/null
+++ b/internal/cli/utils/uuid.go
@@ -0,0 +1,7 @@
+package utils
+
+import "github.com/google/uuid"
+
+var NewUUID = func() string {
+ return uuid.New().String()
+}
diff --git a/kubectl/internal/client/client.go b/internal/client/client.go
similarity index 69%
rename from kubectl/internal/client/client.go
rename to internal/client/client.go
index a4c81d8e..8311805a 100644
--- a/kubectl/internal/client/client.go
+++ b/internal/client/client.go
@@ -3,11 +3,16 @@ package client
import (
"context"
"fmt"
+ "io"
"time"
meta "k8s.io/apimachinery/pkg/api/meta"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
+ "k8s.io/apimachinery/pkg/watch"
"k8s.io/cli-runtime/pkg/resource"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
@@ -31,9 +36,12 @@ func init() {
var _ Interface = &Client{}
type Interface interface {
- PortForwardNotebook(ctx context.Context, verbose bool, nb *apiv1.Notebook, ready chan struct{}) error
+ PortForward(ctx context.Context, logger io.Writer, podRef types.NamespacedName, ports ForwardedPorts, ready chan struct{}) error
Resource(obj Object) (*Resource, error)
- SyncFilesFromNotebook(context.Context, *apiv1.Notebook, string) error
+ SyncFilesFromNotebook(context.Context, *apiv1.Notebook, string,
+ io.Writer,
+ func(file string, complete bool, err error),
+ ) error
}
func NewClient(inf kubernetes.Interface, cfg *rest.Config) Interface {
@@ -98,13 +106,15 @@ func newRestClient(restConfig *rest.Config, gv schema.GroupVersion) (rest.Interf
return rest.RESTClientFor(restConfig)
}
-func (r *Resource) WaitReady(ctx context.Context, obj Object) error {
+func (r *Resource) WaitReady(ctx context.Context, obj Object, progressF func(Object)) error {
if err := wait.PollImmediateInfiniteWithContext(ctx, time.Second,
func(ctx context.Context) (bool, error) {
fetched, err := r.Get(obj.GetNamespace(), obj.GetName())
if err != nil {
return false, err
}
+ fetched.GetObjectKind().SetGroupVersionKind(obj.GetObjectKind().GroupVersionKind())
+ progressF(fetched.(Object))
readyable, ok := fetched.(interface{ GetStatusReady() bool })
if !ok {
return false, fmt.Errorf("object is not readyable: %T", fetched)
@@ -118,3 +128,20 @@ func (r *Resource) WaitReady(ctx context.Context, obj Object) error {
return nil
}
+
+func (r *Resource) Watch(ctx context.Context, namespace string, obj Object, opts *metav1.ListOptions) (watch.Interface, error) {
+ opts.Watch = true
+ if obj != nil && obj.GetName() != "" {
+ opts.ResourceVersion = obj.GetResourceVersion()
+ opts.FieldSelector = fields.OneTermEqualSelector("metadata.name", obj.GetName()).String()
+ }
+
+ // NOTE: The r.Helper.Watch() method does not support passing a context, calling the code
+ // below instead (it was pulled from the Helper implementation).
+ w := r.RESTClient.Get().
+ NamespaceIfScoped(namespace, r.NamespaceScoped).
+ Resource(r.Resource).
+ VersionedParams(opts, metav1.ParameterCodec)
+
+ return w.Watch(ctx)
+}
diff --git a/kubectl/internal/cp/kubectl.go b/internal/client/cp/kubectl.go
similarity index 100%
rename from kubectl/internal/cp/kubectl.go
rename to internal/client/cp/kubectl.go
diff --git a/kubectl/internal/client/decode_encode.go b/internal/client/decode_encode.go
similarity index 91%
rename from kubectl/internal/client/decode_encode.go
rename to internal/client/decode_encode.go
index de554efa..a39129fd 100644
--- a/kubectl/internal/client/decode_encode.go
+++ b/internal/client/decode_encode.go
@@ -9,12 +9,13 @@ func Decode(data []byte) (Object, error) {
decoder := scheme.Codecs.UniversalDeserializer()
runtimeObject, gvk, err := decoder.Decode(data, nil, nil)
+ if gvk == nil {
+ return nil, nil
+ }
if err != nil {
return nil, err
}
- _ = gvk
-
return runtimeObject.(Object), nil
}
diff --git a/internal/client/notebook.go b/internal/client/notebook.go
new file mode 100644
index 00000000..235d2d6f
--- /dev/null
+++ b/internal/client/notebook.go
@@ -0,0 +1,86 @@
+package client
+
+import (
+ "fmt"
+
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+
+ apiv1 "github.com/substratusai/substratus/api/v1"
+)
+
+func PodForNotebook(nb *apiv1.Notebook) types.NamespacedName {
+ // TODO: Pull Pod info from status of Notebook.
+ return types.NamespacedName{
+ Namespace: nb.GetNamespace(),
+ Name: nb.GetName() + "-notebook",
+ }
+}
+
+func NotebookForObject(obj Object) (*apiv1.Notebook, error) {
+ var nb *apiv1.Notebook
+
+ switch obj := obj.DeepCopyObject().(type) {
+ case *apiv1.Notebook:
+ nb = obj
+
+ case *apiv1.Model:
+ nb = &apiv1.Notebook{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: obj.Name + "-model",
+ Namespace: obj.Namespace,
+ },
+ Spec: apiv1.NotebookSpec{
+ Image: obj.Spec.Image,
+ Env: obj.Spec.Env,
+ Params: obj.Spec.Params,
+ Model: obj.Spec.Model,
+ Dataset: obj.Spec.Dataset,
+ Resources: obj.Spec.Resources,
+ },
+ }
+
+ case *apiv1.Server:
+ nb = &apiv1.Notebook{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: obj.Name + "-server",
+ Namespace: obj.Namespace,
+ },
+ Spec: apiv1.NotebookSpec{
+ Image: obj.Spec.Image,
+ Env: obj.Spec.Env,
+ Params: obj.Spec.Params,
+ Model: &obj.Spec.Model,
+ Resources: obj.Spec.Resources,
+ },
+ }
+
+ case *apiv1.Dataset:
+ nb = &apiv1.Notebook{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: obj.Name + "-dataset",
+ Namespace: obj.Namespace,
+ },
+ Spec: apiv1.NotebookSpec{
+ Image: obj.Spec.Image,
+ Env: obj.Spec.Env,
+ Params: obj.Spec.Params,
+ Resources: obj.Spec.Resources,
+ },
+ }
+
+ default:
+ return nil, fmt.Errorf("unknown object type: %T", obj)
+ }
+
+ // "This field is managed by the API server and should not be changed by the user."
+ // https://kubernetes.io/docs/reference/using-api/server-side-apply/#field-management
+ nb.ObjectMeta.ManagedFields = nil
+
+ nb.TypeMeta = metav1.TypeMeta{
+ APIVersion: "substratus.ai/v1",
+ Kind: "Notebook",
+ }
+
+ return nb, nil
+}
diff --git a/kubectl/internal/client/port_forward.go b/internal/client/port_forward.go
similarity index 53%
rename from kubectl/internal/client/port_forward.go
rename to internal/client/port_forward.go
index 9541fc38..71b45444 100644
--- a/kubectl/internal/client/port_forward.go
+++ b/internal/client/port_forward.go
@@ -6,26 +6,19 @@ import (
"io"
"net/http"
"net/url"
- "os"
"strings"
- apiv1 "github.com/substratusai/substratus/api/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/portforward"
"k8s.io/client-go/transport/spdy"
)
-func podForNotebook(nb *apiv1.Notebook) types.NamespacedName {
- // TODO: Pull Pod info from status of Notebook.
- return types.NamespacedName{
- Namespace: nb.GetNamespace(),
- Name: nb.GetName() + "-notebook",
- }
+type ForwardedPorts struct {
+ Local int
+ Pod int
}
-func (c *Client) PortForwardNotebook(ctx context.Context, verbose bool, nb *apiv1.Notebook, ready chan struct{}) error {
- // TODO: Pull Pod info from status of Notebook.
- podRef := podForNotebook(nb)
+func (c *Client) PortForward(ctx context.Context, logger io.Writer, podRef types.NamespacedName, ports ForwardedPorts, ready chan struct{}) error {
path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward",
podRef.Namespace, podRef.Name)
hostIP := strings.TrimLeft(c.Config.Host, "https://")
@@ -35,18 +28,15 @@ func (c *Client) PortForwardNotebook(ctx context.Context, verbose bool, nb *apiv
return err
}
- // TODO: Use an available local port, or allow it to be overridden.
- localPort, podPort := 8888, 8888
-
dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, http.MethodPost, &url.URL{Scheme: "https", Path: path, Host: hostIP})
var stdout, stderr io.Writer
- if verbose {
- stdout, stderr = os.Stdout, os.Stderr
+ if logger != nil {
+ stdout, stderr = logger, logger
} else {
stdout, stderr = io.Discard, io.Discard
}
- fw, err := portforward.New(dialer, []string{fmt.Sprintf("%d:%d", localPort, podPort)}, ctx.Done(), ready, stdout, stderr)
+ fw, err := portforward.New(dialer, []string{fmt.Sprintf("%d:%d", ports.Local, ports.Pod)}, ctx.Done(), ready, stdout, stderr)
if err != nil {
return err
}
diff --git a/kubectl/internal/client/sync.go b/internal/client/sync.go
similarity index 82%
rename from kubectl/internal/client/sync.go
rename to internal/client/sync.go
index 4c1f1bc8..9219a682 100644
--- a/kubectl/internal/client/sync.go
+++ b/internal/client/sync.go
@@ -8,6 +8,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "log"
"net/http"
"os"
"path/filepath"
@@ -19,14 +20,16 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/remotecommand"
- "k8s.io/klog/v2"
apiv1 "github.com/substratusai/substratus/api/v1"
- "github.com/substratusai/substratus/kubectl/internal/cp"
+ "github.com/substratusai/substratus/internal/client/cp"
)
-func (c *Client) SyncFilesFromNotebook(ctx context.Context, nb *apiv1.Notebook, localDir string) error {
- podRef := podForNotebook(nb)
+func (c *Client) SyncFilesFromNotebook(ctx context.Context, nb *apiv1.Notebook, localDir string,
+ logger io.Writer,
+ progressF func(file string, complete bool, err error),
+) error {
+ podRef := PodForNotebook(nb)
const containerName = "notebook"
cacheDir, err := os.UserCacheDir()
@@ -69,57 +72,64 @@ func (c *Client) SyncFilesFromNotebook(ctx context.Context, nb *apiv1.Notebook,
go func() {
defer func() {
wg.Done()
- klog.V(2).Info("File sync loop: Done.")
+ log.Print("File sync loop: Done.")
}()
- klog.V(2).Info("Reading events...")
+ log.Print("Reading events...")
scanner := bufio.NewScanner(r)
for scanner.Scan() {
eventLine := scanner.Bytes()
var event NBWatchEvent
if err := json.Unmarshal(eventLine, &event); err != nil {
- klog.Errorf("Failed to unmarshal nbevent: %w", err)
+ log.Printf("Non-json nbevent: %v", string(eventLine))
continue
}
- relPath, err := filepath.Rel("/content/src", event.Path)
+ relPath, err := filepath.Rel("/content", event.Path)
if err != nil {
- klog.Errorf("Failed to determine relative path: %w", err)
+ log.Printf("Failed to determine relative path: %v", err)
continue
}
- localPath := filepath.Join(localDir, "src", relPath)
+ localPath := filepath.Join(localDir, relPath)
// Possible: CREATE, REMOVE, WRITE, RENAME, CHMOD
if event.Op == "WRITE" || event.Op == "CREATE" {
// NOTE: A long-running port-forward might be more performant here.
+ progressF(event.Path, false, nil)
if err := cp.FromPod(ctx, event.Path, localPath, podRef, containerName); err != nil {
- klog.Errorf("Sync: failed to copy: %w", err)
+ log.Printf("Sync: failed to copy: %v", err)
+ progressF(event.Path, false, err)
continue
}
+ progressF(event.Path, true, nil)
} else if event.Op == "REMOVE" || event.Op == "RENAME" {
+ progressF(event.Path, false, nil)
if err := os.Remove(localPath); err != nil {
- klog.Errorf("Sync: failed to remove: %w", err)
+ log.Printf("Sync: failed to remove: %v", err)
+ progressF(event.Path, false, err)
continue
}
+ progressF(event.Path, true, nil)
}
}
if err := scanner.Err(); err != nil {
- klog.Error("Error reading from buffer:", err)
+ log.Print("Error reading from buffer:", err)
return
}
- klog.V(2).Info("Done reading events.")
+ log.Print("Done reading events.")
}()
- if err := c.exec(ctx, podRef, "/tmp/nbwatch", nil, w, os.Stderr); err != nil {
+ log.Print("Executing nbwatch...")
+ if err := c.exec(ctx, podRef, "/tmp/nbwatch", nil, w, logger); err != nil {
w.Close()
return fmt.Errorf("exec: nbwatch: %w", err)
}
- klog.V(2).Info("Waiting for file sync loop to finish...")
+ log.Print("Waiting for file sync loop to finish...")
wg.Wait()
- klog.V(2).Info("Done waiting for file sync loop to finish.")
+ log.Print("Done waiting for file sync loop to finish.")
return nil
}
@@ -185,10 +195,10 @@ func getContainerTools(ctx context.Context, dir, targetOS string) error {
}
versionStr := strings.TrimSpace(string(version))
if versionStr == Version {
- klog.V(1).Infof("Version (%q) matches for container-tools, skipping download.", Version)
+ log.Printf("Version (%q) matches for container-tools, skipping download.", Version)
return nil
} else {
- klog.V(1).Infof("Version (%q) does not match version.txt: %q", Version, versionStr)
+ log.Printf("Version (%q) does not match version.txt: %q", Version, versionStr)
}
}
@@ -216,7 +226,7 @@ func getContainerTools(ctx context.Context, dir, targetOS string) error {
func getContainerToolsRelease(ctx context.Context, dir, targetOS, targetArch string) error {
releaseURL := fmt.Sprintf("https://github.com/substratusai/substratus/releases/download/v%s/container-tools-%s-%s.tar.gz", Version, targetOS, targetArch)
- klog.V(1).Infof("Downloading: %s", releaseURL)
+ log.Printf("Downloading: %s", releaseURL)
req, err := http.NewRequestWithContext(ctx, "GET", releaseURL, nil)
if err != nil {
@@ -246,7 +256,7 @@ func getContainerToolsRelease(ctx context.Context, dir, targetOS, targetArch str
}
dest := filepath.Join(dir, hdr.Name)
- klog.V(1).Infof("Writing %s", dest)
+ log.Printf("Writing %s", dest)
f, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode))
if err != nil {
return fmt.Errorf("creating file: %w", err)
diff --git a/kubectl/internal/client/upload.go b/internal/client/upload.go
similarity index 84%
rename from kubectl/internal/client/upload.go
rename to internal/client/upload.go
index df8e533e..23ff3d6a 100644
--- a/kubectl/internal/client/upload.go
+++ b/internal/client/upload.go
@@ -11,22 +11,20 @@ import (
"errors"
"fmt"
"io"
+ "log"
"net/http"
"os"
"path/filepath"
"strings"
"time"
- "k8s.io/utils/ptr"
-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
- "k8s.io/apimachinery/pkg/watch"
-
"k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/watch"
+ "k8s.io/utils/ptr"
apiv1 "github.com/substratusai/substratus/api/v1"
- "k8s.io/klog/v2"
)
var httpClient = &http.Client{}
@@ -37,7 +35,7 @@ type Tarball struct {
MD5Checksum string
}
-func PrepareImageTarball(ctx context.Context, buildPath string) (*Tarball, error) {
+func PrepareImageTarball(ctx context.Context, buildPath string, progressF func(file string)) (*Tarball, error) {
exists, err := fileExists(filepath.Join(buildPath, "Dockerfile"))
if err != nil {
return nil, fmt.Errorf("checking if Dockerfile exists: %w", err)
@@ -46,13 +44,13 @@ func PrepareImageTarball(ctx context.Context, buildPath string) (*Tarball, error
return nil, fmt.Errorf("path does not contain Dockerfile: %s", buildPath)
}
- tmpDir, err := os.MkdirTemp("/tmp", "substratus-kubectl-upload")
+ tmpDir, err := os.MkdirTemp("/tmp", "substratus-upload")
if err != nil {
return nil, fmt.Errorf("failed to create temp dir: %w", err)
}
tarPath := filepath.Join(tmpDir, "/archive.tar.gz")
- err = tarGz(ctx, buildPath, tarPath)
+ err = tarGz(ctx, buildPath, tarPath, progressF)
if err != nil {
return nil, fmt.Errorf("failed to create a tar.gz of the directory: %w", err)
}
@@ -125,7 +123,7 @@ func (r *Resource) Apply(obj Object, force bool) error {
return nil
}
-func (r *Resource) Upload(ctx context.Context, obj Object, tb *Tarball) error {
+func (r *Resource) Upload(ctx context.Context, obj Object, tb *Tarball, progressF func(float64)) error {
// NOTE: The r.Helper.WatchSingle() method does not support passing a context, calling the code
// below instead (it was pulled from the Helper implementation).
watcher, err := r.RESTClient.Get().
@@ -156,7 +154,7 @@ loop:
if status.StoredMD5Checksum == tb.MD5Checksum {
// This is an edge-case where the controller found a matching upload
// that already existed in storage.
- klog.V(1).Infof("upload already exists in storage with md5 checksum: %s, skipping upload", status.StoredMD5Checksum)
+ log.Printf("upload already exists in storage with md5 checksum: %s, skipping upload", status.StoredMD5Checksum)
return nil
}
if status.SignedURL != "" && status.RequestID == spec.RequestID {
@@ -179,7 +177,7 @@ loop:
}
}
- if err := uploadTarball(tb, uploadURL); err != nil {
+ if err := uploadTarball(tb, uploadURL, progressF); err != nil {
return fmt.Errorf("uploading tarball: %w", err)
}
@@ -208,7 +206,7 @@ func calculateMD5(path string) (string, error) {
return fmt.Sprintf("%x", hash.Sum(nil)), nil
}
-func tarGz(ctx context.Context, src, dst string) error {
+func tarGz(ctx context.Context, src, dst string, progressF func(string)) error {
tarFile, err := os.Create(dst)
if err != nil {
return fmt.Errorf("failed to create tarFile: %w", err)
@@ -221,9 +219,8 @@ func tarGz(ctx context.Context, src, dst string) error {
tarWriter := tar.NewWriter(gzWriter)
defer tarWriter.Close()
- // TODO(bjb): #121 read .dockerignore if it exists, exclude those files
- return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
- klog.V(4).Infof("Tarring: %v", path)
+ err = filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+ log.Printf("Tarring: %v", path)
if err := ctx.Err(); err != nil {
return err
}
@@ -272,8 +269,15 @@ func tarGz(ctx context.Context, src, dst string) error {
}
}
+ progressF(path)
+
return nil
})
+ if err != nil {
+ return err
+ }
+
+ return nil
}
func fileExists(filename string) (bool, error) {
@@ -287,7 +291,21 @@ func fileExists(filename string) (bool, error) {
return !info.IsDir(), nil
}
-func uploadTarball(tarball *Tarball, url string) error {
+type progressReader struct {
+ totalRead int64
+ total int64
+ r io.Reader
+ f func(float64)
+}
+
+func (r *progressReader) Read(p []byte) (int, error) {
+ n, err := r.r.Read(p)
+ r.totalRead += int64(n)
+ r.f(float64(r.totalRead) / float64(r.total))
+ return n, err
+}
+
+func uploadTarball(tarball *Tarball, url string, progressF func(float64)) error {
data, err := hex.DecodeString(tarball.MD5Checksum)
if err != nil {
return fmt.Errorf("failed to decode hex checksum: %w", err)
@@ -300,8 +318,17 @@ func uploadTarball(tarball *Tarball, url string) error {
}
defer file.Close()
- klog.V(2).Infof("uploading tarball to: %s", url)
- req, err := http.NewRequest(http.MethodPut, url, file)
+ stat, err := file.Stat()
+ if err != nil {
+ return fmt.Errorf("stat: %w", err)
+ }
+
+ log.Printf("uploading tarball to: %s", url)
+ req, err := http.NewRequest(http.MethodPut, url, &progressReader{
+ total: stat.Size(),
+ r: file,
+ f: progressF,
+ })
if err != nil {
return fmt.Errorf("tar upload: %w", err)
}
@@ -319,6 +346,6 @@ func uploadTarball(tarball *Tarball, url string) error {
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected response status: %d", resp.StatusCode)
}
- klog.V(1).Info("successfully uploaded tarball")
+ log.Print("successfully uploaded tarball")
return nil
}
diff --git a/internal/cloud/cloud.go b/internal/cloud/cloud.go
index 54c80653..d7dd6aef 100644
--- a/internal/cloud/cloud.go
+++ b/internal/cloud/cloud.go
@@ -8,10 +8,11 @@ import (
"cloud.google.com/go/compute/metadata"
"github.com/go-playground/validator/v10"
"github.com/sethvargo/go-envconfig"
- apiv1 "github.com/substratusai/substratus/api/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
+
+ apiv1 "github.com/substratusai/substratus/api/v1"
)
const CloudEnvVar = "CLOUD"
diff --git a/internal/controller/build_reconciler.go b/internal/controller/build_reconciler.go
index 07795603..796ed945 100644
--- a/internal/controller/build_reconciler.go
+++ b/internal/controller/build_reconciler.go
@@ -8,10 +8,6 @@ import (
"strings"
"time"
- apiv1 "github.com/substratusai/substratus/api/v1"
- "github.com/substratusai/substratus/internal/cloud"
- "github.com/substratusai/substratus/internal/resources"
- "github.com/substratusai/substratus/internal/sci"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -23,6 +19,11 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/log"
+
+ apiv1 "github.com/substratusai/substratus/api/v1"
+ "github.com/substratusai/substratus/internal/cloud"
+ "github.com/substratusai/substratus/internal/resources"
+ "github.com/substratusai/substratus/internal/sci"
)
const latestUploadPath = "uploads/latest.tar.gz"
@@ -282,7 +283,8 @@ func (r *BuildReconciler) gitBuildJob(ctx context.Context, obj BuildableObject)
// Disable compressed caching to decrease memory usage.
// (See https://github.com/GoogleContainerTools/kaniko/blob/main/README.md#flag---compressed-caching)
"--compressed-caching=false",
- "--log-format=text",
+ "--log-format=color",
+ "--log-timestamp=false",
}
var initContainers []corev1.Container
@@ -327,8 +329,10 @@ func (r *BuildReconciler) gitBuildJob(ctx context.Context, obj BuildableObject)
corev1.Container{
Name: "gcp-workload-identity-readiness-check",
Image: "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine",
- Args: []string{"/bin/bash", "-c",
- "curl -sS -H 'Metadata-Flavor: Google' 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token' --retry 30 --retry-connrefused --retry-max-time 60 --connect-timeout 3 --fail --retry-all-errors > /dev/null && exit 0 || echo 'Retry limit exceeded. Failed to wait for metadata server to be available. Check if the gke-metadata-server Pod in the kube-system namespace is healthy.' >&2; exit 1"},
+ Args: []string{
+ "/bin/bash", "-c",
+ "curl -sS -H 'Metadata-Flavor: Google' 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token' --retry 30 --retry-connrefused --retry-max-time 60 --connect-timeout 3 --fail --retry-all-errors > /dev/null && exit 0 || echo 'Retry limit exceeded. Failed to wait for metadata server to be available. Check if the gke-metadata-server Pod in the kube-system namespace is healthy.' >&2; exit 1",
+ },
})
}
@@ -364,6 +368,10 @@ func (r *BuildReconciler) gitBuildJob(ctx context.Context, obj BuildableObject)
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: annotations,
+ Labels: map[string]string{
+ strings.ToLower(r.Kind): obj.GetName(),
+ "role": "build",
+ },
},
Spec: corev1.PodSpec{
InitContainers: initContainers,
@@ -408,7 +416,8 @@ func (r *BuildReconciler) storageBuildJob(ctx context.Context, obj BuildableObje
// Disable compressed caching to decrease memory usage.
// (See https://github.com/GoogleContainerTools/kaniko/blob/main/README.md#flag---compressed-caching)
"--compressed-caching=false",
- "--log-format=text",
+ "--log-format=color",
+ "--log-timestamp=false",
}
var initContainers []corev1.Container
@@ -466,8 +475,10 @@ func (r *BuildReconciler) storageBuildJob(ctx context.Context, obj BuildableObje
corev1.Container{
Name: "gcp-workload-identity-readiness-check",
Image: "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine",
- Args: []string{"/bin/bash", "-c",
- "curl -sS -H 'Metadata-Flavor: Google' 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token' --retry 30 --retry-connrefused --retry-max-time 60 --connect-timeout 3 --fail --retry-all-errors > /dev/null && exit 0 || echo 'Retry limit exceeded. Failed to wait for metadata server to be available. Check if the gke-metadata-server Pod in the kube-system namespace is healthy.' >&2; exit 1"},
+ Args: []string{
+ "/bin/bash", "-c",
+ "curl -sS -H 'Metadata-Flavor: Google' 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token' --retry 30 --retry-connrefused --retry-max-time 60 --connect-timeout 3 --fail --retry-all-errors > /dev/null && exit 0 || echo 'Retry limit exceeded. Failed to wait for metadata server to be available. Check if the gke-metadata-server Pod in the kube-system namespace is healthy.' >&2; exit 1",
+ },
})
}
@@ -487,6 +498,10 @@ func (r *BuildReconciler) storageBuildJob(ctx context.Context, obj BuildableObje
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: podAnnotations,
+ Labels: map[string]string{
+ strings.ToLower(r.Kind): obj.GetName(),
+ "role": "build",
+ },
},
Spec: corev1.PodSpec{
InitContainers: initContainers,
diff --git a/internal/controller/dataset_controller.go b/internal/controller/dataset_controller.go
index 3f44dd6c..514bbd08 100644
--- a/internal/controller/dataset_controller.go
+++ b/internal/controller/dataset_controller.go
@@ -103,25 +103,38 @@ func (r *DatasetReconciler) reconcileData(ctx context.Context, dataset *apiv1.Da
return result{}, nil
}
- dataset.Status.Ready = false
- meta.SetStatusCondition(dataset.GetConditions(), metav1.Condition{
- Type: apiv1.ConditionLoaded,
- Status: metav1.ConditionFalse,
- Reason: apiv1.ReasonJobNotComplete,
- ObservedGeneration: dataset.Generation,
- Message: "Waiting for data loader Job to complete",
- })
if err := r.Status().Update(ctx, dataset); err != nil {
return result{}, fmt.Errorf("updating status: %w", err)
}
- if result, err := reconcileJob(ctx, r.Client, loadJob, apiv1.ConditionLoaded); !result.success {
- return result, err
+ jobResult, err := reconcileJob(ctx, r.Client, loadJob)
+ if !jobResult.success {
+ dataset.Status.Ready = false
+ if !jobResult.failure {
+ meta.SetStatusCondition(dataset.GetConditions(), metav1.Condition{
+ Type: apiv1.ConditionComplete,
+ Status: metav1.ConditionFalse,
+ Reason: apiv1.ReasonJobNotComplete,
+ ObservedGeneration: dataset.Generation,
+ Message: "Waiting for data loader Job to complete",
+ })
+ } else {
+ meta.SetStatusCondition(dataset.GetConditions(), metav1.Condition{
+ Type: apiv1.ConditionComplete,
+ Status: metav1.ConditionFalse,
+ Reason: apiv1.ReasonJobFailed,
+ ObservedGeneration: dataset.Generation,
+ })
+ }
+ if err := r.Status().Update(ctx, dataset); err != nil {
+ return result{}, fmt.Errorf("updating status: %w", err)
+ }
+ return jobResult, err
}
dataset.Status.Ready = true
meta.SetStatusCondition(dataset.GetConditions(), metav1.Condition{
- Type: apiv1.ConditionLoaded,
+ Type: apiv1.ConditionComplete,
Status: metav1.ConditionTrue,
Reason: apiv1.ReasonJobComplete,
ObservedGeneration: dataset.Generation,
@@ -146,11 +159,16 @@ func (r *DatasetReconciler) loadJob(ctx context.Context, dataset *apiv1.Dataset)
Namespace: dataset.Namespace,
},
Spec: batchv1.JobSpec{
+ BackoffLimit: ptr.To(int32(2)), // TotalRetries = BackoffLimit + 1
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"kubectl.kubernetes.io/default-container": containerName,
},
+ Labels: map[string]string{
+ "dataset": dataset.Name,
+ "role": "run",
+ },
},
Spec: corev1.PodSpec{
SecurityContext: &corev1.PodSecurityContext{
@@ -176,10 +194,9 @@ func (r *DatasetReconciler) loadJob(ctx context.Context, dataset *apiv1.Dataset)
}
if err := r.Cloud.MountBucket(&job.Spec.Template.ObjectMeta, &job.Spec.Template.Spec, dataset, cloud.MountBucketConfig{
- Name: "dataset",
+ Name: "artifacts",
Mounts: []cloud.BucketMount{
- {BucketSubdir: "data", ContentSubdir: "data"},
- {BucketSubdir: "logs", ContentSubdir: "logs"},
+ {BucketSubdir: "artifacts", ContentSubdir: "artifacts"},
},
Container: containerName,
ReadOnly: false,
diff --git a/internal/controller/dataset_controller_test.go b/internal/controller/dataset_controller_test.go
index 2e7ad5f6..0ecafd49 100644
--- a/internal/controller/dataset_controller_test.go
+++ b/internal/controller/dataset_controller_test.go
@@ -6,7 +6,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- apiv1 "github.com/substratusai/substratus/api/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
@@ -14,6 +13,8 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/controller-runtime/pkg/client"
+
+ apiv1 "github.com/substratusai/substratus/api/v1"
)
func TestDataset(t *testing.T) {
@@ -68,9 +69,8 @@ func testDatasetLoad(t *testing.T, dataset *apiv1.Dataset) {
require.EventuallyWithT(t, func(t *assert.CollectT) {
err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dataset), dataset)
assert.NoError(t, err, "getting the dataset")
- assert.True(t, meta.IsStatusConditionTrue(dataset.Status.Conditions, apiv1.ConditionLoaded))
+ assert.True(t, meta.IsStatusConditionTrue(dataset.Status.Conditions, apiv1.ConditionComplete))
assert.True(t, dataset.Status.Ready)
}, timeout, interval, "waiting for the dataset to be ready")
require.Contains(t, dataset.Status.Artifacts.URL, "gs://test-artifact-bucket")
-
}
diff --git a/internal/controller/main_test.go b/internal/controller/main_test.go
index b00cddec..c86c45d4 100644
--- a/internal/controller/main_test.go
+++ b/internal/controller/main_test.go
@@ -11,26 +11,24 @@ import (
"testing"
"time"
- ctrl "sigs.k8s.io/controller-runtime"
-
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes/scheme"
+ ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
apiv1 "github.com/substratusai/substratus/api/v1"
"github.com/substratusai/substratus/internal/cloud"
"github.com/substratusai/substratus/internal/controller"
"github.com/substratusai/substratus/internal/sci"
- //+kubebuilder:scaffold:imports
)
const (
@@ -46,10 +44,10 @@ var (
)
func TestMain(m *testing.M) {
- //var buf bytes.Buffer
+ // var buf bytes.Buffer
logf.SetLogger(zap.New(
zap.UseDevMode(true),
- //zap.WriteTo(&buf),
+ // zap.WriteTo(&buf),
))
ctx, cancel = context.WithCancel(context.TODO())
@@ -87,8 +85,8 @@ func TestMain(m *testing.M) {
sciClient := &sci.FakeSCIControllerClient{}
- //runtimeMgr, err := controller.NewRuntimeManager(controller.GPUTypeNvidiaL4)
- //requireNoError(err)
+ // runtimeMgr, err := controller.NewRuntimeManager(controller.GPUTypeNvidiaL4)
+ // requireNoError(err)
err = (&controller.ModelReconciler{
Client: mgr.GetClient(),
@@ -247,6 +245,12 @@ func testParamsConfigMap(t *testing.T, obj testObject, kind string, content stri
func fakeJobComplete(t *testing.T, job *batchv1.Job) {
updated := job.DeepCopy()
updated.Status.Succeeded = 1
+ updated.Status.Conditions = []batchv1.JobCondition{
+ {
+ Type: batchv1.JobComplete,
+ Status: corev1.ConditionTrue,
+ },
+ }
require.NoError(t, k8sClient.Status().Patch(ctx, updated, client.MergeFrom(job)), "patching the job with completed count")
}
diff --git a/internal/controller/manager.go b/internal/controller/manager.go
index 70b745cd..80882cd8 100644
--- a/internal/controller/manager.go
+++ b/internal/controller/manager.go
@@ -4,17 +4,18 @@ import (
"context"
"fmt"
- apiv1 "github.com/substratusai/substratus/api/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/manager"
+
+ apiv1 "github.com/substratusai/substratus/api/v1"
)
const (
notebookModelIndex = "spec.model.name"
notebookDatasetIndex = "spec.dataset.name"
- modelBaseModelIndex = "spec.baseModel.name"
- modelTrainingDatasetIndex = "spec.trainingDataset.name"
+ modelModelIndex = "spec.model.name"
+ modelDatasetIndex = "spec.dataset.name"
modelServerModelIndex = "spec.model.name"
)
@@ -40,22 +41,22 @@ func SetupIndexes(mgr manager.Manager) error {
return fmt.Errorf("notebook: %w", err)
}
- if err := mgr.GetFieldIndexer().IndexField(context.Background(), &apiv1.Model{}, modelBaseModelIndex, func(rawObj client.Object) []string {
+ if err := mgr.GetFieldIndexer().IndexField(context.Background(), &apiv1.Model{}, modelModelIndex, func(rawObj client.Object) []string {
model := rawObj.(*apiv1.Model)
- if model.Spec.BaseModel == nil {
+ if model.Spec.Model == nil {
return []string{}
}
- return []string{model.Spec.BaseModel.Name}
+ return []string{model.Spec.Model.Name}
}); err != nil {
return fmt.Errorf("model: %w", err)
}
- if err := mgr.GetFieldIndexer().IndexField(context.Background(), &apiv1.Model{}, modelTrainingDatasetIndex, func(rawObj client.Object) []string {
+ if err := mgr.GetFieldIndexer().IndexField(context.Background(), &apiv1.Model{}, modelDatasetIndex, func(rawObj client.Object) []string {
model := rawObj.(*apiv1.Model)
- if model.Spec.TrainingDataset == nil {
+ if model.Spec.Dataset == nil {
return []string{}
}
- return []string{model.Spec.TrainingDataset.Name}
+ return []string{model.Spec.Dataset.Name}
}); err != nil {
return fmt.Errorf("model: %w", err)
}
diff --git a/internal/controller/model_controller.go b/internal/controller/model_controller.go
index f373a537..6c6f7c5c 100644
--- a/internal/controller/model_controller.go
+++ b/internal/controller/model_controller.go
@@ -90,14 +90,14 @@ func (r *ModelReconciler) reconcileModel(ctx context.Context, model *apiv1.Model
}
var baseModel *apiv1.Model
- if model.Spec.BaseModel != nil {
+ if model.Spec.Model != nil {
baseModel = &apiv1.Model{}
- if err := r.Client.Get(ctx, types.NamespacedName{Namespace: model.Namespace, Name: model.Spec.BaseModel.Name}, baseModel); err != nil {
+ if err := r.Client.Get(ctx, types.NamespacedName{Namespace: model.Namespace, Name: model.Spec.Model.Name}, baseModel); err != nil {
if apierrors.IsNotFound(err) {
// Update this Model's status.
model.Status.Ready = false
meta.SetStatusCondition(&model.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionModelled,
+ Type: apiv1.ConditionComplete,
Status: metav1.ConditionFalse,
Reason: apiv1.ReasonBaseModelNotFound,
ObservedGeneration: model.Generation,
@@ -116,7 +116,7 @@ func (r *ModelReconciler) reconcileModel(ctx context.Context, model *apiv1.Model
// Update this Model's status.
model.Status.Ready = false
meta.SetStatusCondition(&model.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionModelled,
+ Type: apiv1.ConditionComplete,
Status: metav1.ConditionFalse,
Reason: apiv1.ReasonBaseModelNotReady,
ObservedGeneration: model.Generation,
@@ -131,14 +131,14 @@ func (r *ModelReconciler) reconcileModel(ctx context.Context, model *apiv1.Model
}
var dataset *apiv1.Dataset
- if model.Spec.TrainingDataset != nil {
+ if model.Spec.Dataset != nil {
dataset = &apiv1.Dataset{}
- if err := r.Client.Get(ctx, types.NamespacedName{Namespace: model.Namespace, Name: model.Spec.TrainingDataset.Name}, dataset); err != nil {
+ if err := r.Client.Get(ctx, types.NamespacedName{Namespace: model.Namespace, Name: model.Spec.Dataset.Name}, dataset); err != nil {
if apierrors.IsNotFound(err) {
// Update this Model's status.
model.Status.Ready = false
meta.SetStatusCondition(&model.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionModelled,
+ Type: apiv1.ConditionComplete,
Status: metav1.ConditionFalse,
Reason: apiv1.ReasonDatasetNotFound,
ObservedGeneration: model.Generation,
@@ -157,7 +157,7 @@ func (r *ModelReconciler) reconcileModel(ctx context.Context, model *apiv1.Model
// Update this Model's status.
model.Status.Ready = false
meta.SetStatusCondition(&model.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionModelled,
+ Type: apiv1.ConditionComplete,
Status: metav1.ConditionFalse,
Reason: apiv1.ReasonDatasetNotReady,
ObservedGeneration: model.Generation,
@@ -178,25 +178,34 @@ func (r *ModelReconciler) reconcileModel(ctx context.Context, model *apiv1.Model
return result{}, nil
}
- model.Status.Ready = false
- meta.SetStatusCondition(model.GetConditions(), metav1.Condition{
- Type: apiv1.ConditionModelled,
- Status: metav1.ConditionFalse,
- Reason: apiv1.ReasonJobNotComplete,
- ObservedGeneration: model.Generation,
- Message: "Waiting for modeller Job to complete",
- })
- if err := r.Status().Update(ctx, model); err != nil {
- return result{}, fmt.Errorf("updating status: %w", err)
- }
-
- if result, err := reconcileJob(ctx, r.Client, modellerJob, apiv1.ConditionModelled); !result.success {
- return result, err
+ jobResult, err := reconcileJob(ctx, r.Client, modellerJob)
+ if !jobResult.success {
+ model.Status.Ready = false
+ if !jobResult.failure {
+ meta.SetStatusCondition(model.GetConditions(), metav1.Condition{
+ Type: apiv1.ConditionComplete,
+ Status: metav1.ConditionFalse,
+ Reason: apiv1.ReasonJobNotComplete,
+ ObservedGeneration: model.Generation,
+ Message: "Waiting for modeller Job to complete",
+ })
+ } else {
+ meta.SetStatusCondition(model.GetConditions(), metav1.Condition{
+ Type: apiv1.ConditionComplete,
+ Status: metav1.ConditionFalse,
+ Reason: apiv1.ReasonJobFailed,
+ ObservedGeneration: model.Generation,
+ })
+ }
+ if err := r.Status().Update(ctx, model); err != nil {
+ return result{}, fmt.Errorf("updating status: %w", err)
+ }
+ return jobResult, err
}
model.Status.Ready = true
meta.SetStatusCondition(model.GetConditions(), metav1.Condition{
- Type: apiv1.ConditionModelled,
+ Type: apiv1.ConditionComplete,
Status: metav1.ConditionTrue,
Reason: apiv1.ReasonJobComplete,
ObservedGeneration: model.Generation,
@@ -230,7 +239,7 @@ func (r *ModelReconciler) findModelsForBaseModel(ctx context.Context, obj client
var models apiv1.ModelList
if err := r.List(ctx, &models,
- client.MatchingFields{modelBaseModelIndex: model.Name},
+ client.MatchingFields{modelModelIndex: model.Name},
client.InNamespace(obj.GetNamespace()),
); err != nil {
log.Log.Error(err, "unable to list models for base model")
@@ -254,7 +263,7 @@ func (r *ModelReconciler) findModelsForDataset(ctx context.Context, obj client.O
var models apiv1.ModelList
if err := r.List(ctx, &models,
- client.MatchingFields{modelTrainingDatasetIndex: dataset.Name},
+ client.MatchingFields{modelDatasetIndex: dataset.Name},
client.InNamespace(obj.GetNamespace()),
); err != nil {
log.Log.Error(err, "unable to list models for dataset")
@@ -282,6 +291,17 @@ func (r *ModelReconciler) modellerJob(ctx context.Context, model, baseModel *api
return nil, fmt.Errorf("resolving env: %w", err)
}
+ // Don't retry expensive Jobs by default.
+ var backoffLimit int32
+ if model.Spec.Resources != nil &&
+ model.Spec.Resources.CPU <= 3 &&
+ model.Spec.Resources.GPU != nil &&
+ model.Spec.Resources.GPU.Count == 0 {
+ // If the Job is not super expensive (No GPUs, low CPUs), then assume
+ // it is an import Job and up the retry count.
+ backoffLimit = 2 // 2 = 3 retries
+ }
+
const containerName = "model"
job = &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
@@ -290,12 +310,17 @@ func (r *ModelReconciler) modellerJob(ctx context.Context, model, baseModel *api
Namespace: model.Namespace,
},
Spec: batchv1.JobSpec{
- BackoffLimit: ptr.To(int32(1)),
+ // TODO: Allow for configurable retries for Jobs that import models...
+ BackoffLimit: ptr.To(backoffLimit),
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"kubectl.kubernetes.io/default-container": containerName,
},
+ Labels: map[string]string{
+ "model": model.Name,
+ "role": "run",
+ },
},
Spec: corev1.PodSpec{
SecurityContext: &corev1.PodSecurityContext{
@@ -321,10 +346,9 @@ func (r *ModelReconciler) modellerJob(ctx context.Context, model, baseModel *api
}
if err := r.Cloud.MountBucket(&job.Spec.Template.ObjectMeta, &job.Spec.Template.Spec, model, cloud.MountBucketConfig{
- Name: "model",
+ Name: "artifacts",
Mounts: []cloud.BucketMount{
- {BucketSubdir: "model", ContentSubdir: "model"},
- {BucketSubdir: "logs", ContentSubdir: "logs"},
+ {BucketSubdir: "artifacts", ContentSubdir: "artifacts"},
},
Container: containerName,
ReadOnly: false,
@@ -336,7 +360,7 @@ func (r *ModelReconciler) modellerJob(ctx context.Context, model, baseModel *api
if err := r.Cloud.MountBucket(&job.Spec.Template.ObjectMeta, &job.Spec.Template.Spec, dataset, cloud.MountBucketConfig{
Name: "dataset",
Mounts: []cloud.BucketMount{
- {BucketSubdir: "data", ContentSubdir: "data"},
+ {BucketSubdir: "artifacts", ContentSubdir: "data"},
},
Container: containerName,
ReadOnly: true,
@@ -347,9 +371,9 @@ func (r *ModelReconciler) modellerJob(ctx context.Context, model, baseModel *api
if baseModel != nil {
if err := r.Cloud.MountBucket(&job.Spec.Template.ObjectMeta, &job.Spec.Template.Spec, baseModel, cloud.MountBucketConfig{
- Name: "basemodel",
+ Name: "model",
Mounts: []cloud.BucketMount{
- {BucketSubdir: "model", ContentSubdir: "saved-model"},
+ {BucketSubdir: "artifacts", ContentSubdir: "model"},
},
Container: containerName,
ReadOnly: true,
diff --git a/internal/controller/model_controller_test.go b/internal/controller/model_controller_test.go
index 930084ed..d478a7d3 100644
--- a/internal/controller/model_controller_test.go
+++ b/internal/controller/model_controller_test.go
@@ -6,7 +6,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- apiv1 "github.com/substratusai/substratus/api/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
@@ -14,6 +13,8 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/ptr"
+
+ apiv1 "github.com/substratusai/substratus/api/v1"
)
func TestModelLoaderFromGit(t *testing.T) {
@@ -58,7 +59,7 @@ func testModelLoad(t *testing.T, model *apiv1.Model) {
require.EventuallyWithT(t, func(t *assert.CollectT) {
err := k8sClient.Get(ctx, types.NamespacedName{Namespace: model.GetNamespace(), Name: model.GetName()}, model)
assert.NoError(t, err, "getting model")
- assert.True(t, meta.IsStatusConditionTrue(model.Status.Conditions, apiv1.ConditionModelled))
+ assert.True(t, meta.IsStatusConditionTrue(model.Status.Conditions, apiv1.ConditionComplete))
assert.True(t, model.Status.Ready)
}, timeout, interval, "waiting for the model to be ready")
require.Contains(t, model.Status.Artifacts.URL, "gs://test-artifact-bucket")
@@ -109,10 +110,10 @@ func TestModelTrainerFromGit(t *testing.T) {
URL: "https://test.com/test/test",
},
},
- BaseModel: &apiv1.ObjectRef{
+ Model: &apiv1.ObjectRef{
Name: baseModel.Name,
},
- TrainingDataset: &apiv1.ObjectRef{
+ Dataset: &apiv1.ObjectRef{
Name: dataset.Name,
},
},
@@ -151,7 +152,7 @@ func testModelTrain(t *testing.T, model *apiv1.Model) {
require.EventuallyWithT(t, func(t *assert.CollectT) {
err := k8sClient.Get(ctx, types.NamespacedName{Namespace: model.GetNamespace(), Name: model.GetName()}, model)
assert.NoError(t, err, "getting model")
- assert.True(t, meta.IsStatusConditionTrue(model.Status.Conditions, apiv1.ConditionModelled))
+ assert.True(t, meta.IsStatusConditionTrue(model.Status.Conditions, apiv1.ConditionComplete))
assert.True(t, model.Status.Ready)
}, timeout, interval, "waiting for the model to be ready")
require.Contains(t, model.Status.Artifacts.URL, "gs://test-artifact-bucket")
diff --git a/internal/controller/notebook_controller.go b/internal/controller/notebook_controller.go
index b9ca6479..4736becd 100644
--- a/internal/controller/notebook_controller.go
+++ b/internal/controller/notebook_controller.go
@@ -134,7 +134,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap
if notebook.IsSuspended() {
notebook.Status.Ready = false
meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionDeployed,
+ Type: apiv1.ConditionServing,
Status: metav1.ConditionFalse,
Reason: apiv1.ReasonSuspended,
ObservedGeneration: notebook.Generation,
@@ -167,7 +167,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap
}
var model *apiv1.Model
- if notebook.Spec.Model != nil {
+ if notebook.Spec.Model != nil && notebook.Spec.Model.Name != "" {
model = &apiv1.Model{}
if err := r.Get(ctx, client.ObjectKey{Name: notebook.Spec.Model.Name, Namespace: notebook.Namespace}, model); err != nil {
if apierrors.IsNotFound(err) {
@@ -175,7 +175,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap
// Update this Model's status.
notebook.Status.Ready = false
meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionDeployed,
+ Type: apiv1.ConditionServing,
Status: metav1.ConditionFalse,
Reason: apiv1.ReasonModelNotFound,
ObservedGeneration: notebook.Generation,
@@ -195,7 +195,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap
notebook.Status.Ready = false
meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionDeployed,
+ Type: apiv1.ConditionServing,
Status: metav1.ConditionFalse,
Reason: apiv1.ReasonModelNotReady,
ObservedGeneration: notebook.Generation,
@@ -210,14 +210,14 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap
}
var dataset *apiv1.Dataset
- if notebook.Spec.Dataset != nil {
+ if notebook.Spec.Dataset != nil && notebook.Spec.Dataset.Name != "" {
dataset = &apiv1.Dataset{}
if err := r.Get(ctx, client.ObjectKey{Name: notebook.Spec.Dataset.Name, Namespace: notebook.Namespace}, dataset); err != nil {
if apierrors.IsNotFound(err) {
log.Error(err, "Dataset not found")
notebook.Status.Ready = false
meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionDeployed,
+ Type: apiv1.ConditionServing,
Status: metav1.ConditionFalse,
Reason: apiv1.ReasonDatasetNotFound,
ObservedGeneration: notebook.Generation,
@@ -236,7 +236,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap
log.Info("Dataset not ready", "dataset", dataset.Name)
notebook.Status.Ready = false
meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionDeployed,
+ Type: apiv1.ConditionServing,
Status: metav1.ConditionFalse,
Reason: apiv1.ReasonDatasetNotReady,
ObservedGeneration: notebook.Generation,
@@ -288,7 +288,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap
if isPodReady(pod) {
notebook.Status.Ready = true
meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionDeployed,
+ Type: apiv1.ConditionServing,
Status: metav1.ConditionTrue,
Reason: apiv1.ReasonPodReady,
ObservedGeneration: notebook.Generation,
@@ -296,7 +296,7 @@ func (r *NotebookReconciler) reconcileNotebook(ctx context.Context, notebook *ap
} else {
notebook.Status.Ready = false
meta.SetStatusCondition(¬ebook.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionDeployed,
+ Type: apiv1.ConditionServing,
Status: metav1.ConditionFalse,
Reason: apiv1.ReasonPodNotReady,
ObservedGeneration: notebook.Generation,
@@ -317,10 +317,22 @@ func nbPodName(nb *apiv1.Notebook) string {
func (r *NotebookReconciler) notebookPod(notebook *apiv1.Notebook, model *apiv1.Model, dataset *apiv1.Dataset) (*corev1.Pod, error) {
const containerName = "notebook"
- envVars, err := resolveEnv(notebook.Spec.Env)
+ cmd := notebook.Spec.Command
+ if cmd == nil {
+ cmd = []string{
+ "jupyter", "lab",
+ "--allow-root",
+ "--ip=0.0.0.0",
+ "--NotebookApp.token=$(NOTEBOOK_TOKEN)",
+ "--notebook-dir=/content",
+ }
+ }
+
+ env, err := resolveEnv(notebook.Spec.Env)
if err != nil {
return nil, fmt.Errorf("resolving env: %w", err)
}
+ env = append(env, corev1.EnvVar{Name: "NOTEBOOK_TOKEN", Value: "default"})
pod := &corev1.Pod{
TypeMeta: metav1.TypeMeta{
@@ -333,6 +345,10 @@ func (r *NotebookReconciler) notebookPod(notebook *apiv1.Notebook, model *apiv1.
Annotations: map[string]string{
"kubectl.kubernetes.io/default-container": containerName,
},
+ Labels: map[string]string{
+ "notebook": notebook.Name,
+ "role": "run",
+ },
},
Spec: corev1.PodSpec{
//SecurityContext: &corev1.PodSecurityContext{
@@ -345,15 +361,16 @@ func (r *NotebookReconciler) notebookPod(notebook *apiv1.Notebook, model *apiv1.
{
Name: containerName,
Image: notebook.GetImage(),
- Command: notebook.Spec.Command,
- //WorkingDir: "/home/jovyan",
+ Command: cmd,
+
+ // WorkingDir: "/home/jovyan",
Ports: []corev1.ContainerPort{
{
Name: "notebook",
ContainerPort: 8888,
},
},
- Env: envVars,
+ Env: env,
// TODO: GPUs
ReadinessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
@@ -392,8 +409,7 @@ func (r *NotebookReconciler) notebookPod(notebook *apiv1.Notebook, model *apiv1.
if err := r.Cloud.MountBucket(&pod.ObjectMeta, &pod.Spec, dataset, cloud.MountBucketConfig{
Name: "dataset",
Mounts: []cloud.BucketMount{
- {BucketSubdir: "data", ContentSubdir: "data"},
- {BucketSubdir: "logs", ContentSubdir: "data-logs"},
+ {BucketSubdir: "artifacts", ContentSubdir: "data"},
},
Container: containerName,
ReadOnly: true,
@@ -404,10 +420,9 @@ func (r *NotebookReconciler) notebookPod(notebook *apiv1.Notebook, model *apiv1.
if model != nil {
if err := r.Cloud.MountBucket(&pod.ObjectMeta, &pod.Spec, model, cloud.MountBucketConfig{
- Name: "basemodel",
+ Name: "model",
Mounts: []cloud.BucketMount{
- {BucketSubdir: "model", ContentSubdir: "saved-model"},
- {BucketSubdir: "logs", ContentSubdir: "saved-model-logs"},
+ {BucketSubdir: "artifacts", ContentSubdir: "model"},
},
Container: containerName,
ReadOnly: true,
@@ -416,6 +431,16 @@ func (r *NotebookReconciler) notebookPod(notebook *apiv1.Notebook, model *apiv1.
}
}
+ // Mounts specific to this Notebook.
+ if err := r.Cloud.MountBucket(&pod.ObjectMeta, &pod.Spec, notebook, cloud.MountBucketConfig{
+ Name: "artifacts",
+ Mounts: []cloud.BucketMount{{BucketSubdir: "artifacts", ContentSubdir: "artifacts"}},
+ Container: containerName,
+ ReadOnly: false,
+ }); err != nil {
+ return nil, fmt.Errorf("mounting notebook: %w", err)
+ }
+
if err := ctrl.SetControllerReference(notebook, pod, r.Scheme); err != nil {
return nil, fmt.Errorf("failed to set controller reference: %w", err)
}
diff --git a/internal/controller/notebook_controller_test.go b/internal/controller/notebook_controller_test.go
index 79f43f72..0b56825c 100644
--- a/internal/controller/notebook_controller_test.go
+++ b/internal/controller/notebook_controller_test.go
@@ -6,7 +6,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- apiv1 "github.com/substratusai/substratus/api/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -14,6 +13,8 @@ import (
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
+
+ apiv1 "github.com/substratusai/substratus/api/v1"
)
func TestNotebookFromGitWithModelOnly(t *testing.T) {
@@ -75,7 +76,7 @@ func TestNotebookFromGitWithModelOnly(t *testing.T) {
require.EventuallyWithT(t, func(t *assert.CollectT) {
err := k8sClient.Get(ctx, client.ObjectKeyFromObject(notebook), notebook)
assert.NoError(t, err, "getting the notebook")
- assert.True(t, meta.IsStatusConditionTrue(notebook.Status.Conditions, apiv1.ConditionDeployed))
+ assert.True(t, meta.IsStatusConditionTrue(notebook.Status.Conditions, apiv1.ConditionServing))
assert.True(t, notebook.Status.Ready)
}, timeout, interval, "waiting for the notebook to be ready")
}
diff --git a/internal/controller/server_controller.go b/internal/controller/server_controller.go
index 0285a3ae..b32b70b7 100644
--- a/internal/controller/server_controller.go
+++ b/internal/controller/server_controller.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
+ "github.com/go-logr/logr"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
@@ -19,7 +20,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
- "github.com/go-logr/logr"
apiv1 "github.com/substratusai/substratus/api/v1"
"github.com/substratusai/substratus/internal/cloud"
"github.com/substratusai/substratus/internal/resources"
@@ -101,7 +101,6 @@ func (r *ServerReconciler) findServersForModel(ctx context.Context, obj client.O
reqs := []reconcile.Request{}
for _, svr := range servers.Items {
-
reqs = append(reqs, reconcile.Request{
NamespacedName: types.NamespacedName{
Name: svr.Name,
@@ -185,7 +184,7 @@ func (r *ServerReconciler) serverDeployment(server *apiv1.Server, model *apiv1.M
if err := r.Cloud.MountBucket(&deploy.Spec.Template.ObjectMeta, &deploy.Spec.Template.Spec, model, cloud.MountBucketConfig{
Name: "model",
Mounts: []cloud.BucketMount{
- {BucketSubdir: "model", ContentSubdir: "saved-model"},
+ {BucketSubdir: "artifacts", ContentSubdir: "model"},
},
Container: containerName,
ReadOnly: true,
@@ -214,7 +213,7 @@ func (r *ServerReconciler) reconcileServer(ctx context.Context, server *apiv1.Se
// Update this Model's status.
server.Status.Ready = false
meta.SetStatusCondition(&server.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionDeployed,
+ Type: apiv1.ConditionServing,
Status: metav1.ConditionFalse,
Reason: apiv1.ReasonModelNotFound,
ObservedGeneration: server.Generation,
@@ -234,7 +233,7 @@ func (r *ServerReconciler) reconcileServer(ctx context.Context, server *apiv1.Se
server.Status.Ready = false
meta.SetStatusCondition(&server.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionDeployed,
+ Type: apiv1.ConditionServing,
Status: metav1.ConditionFalse,
Reason: apiv1.ReasonModelNotReady,
ObservedGeneration: server.Generation,
@@ -281,7 +280,7 @@ func (r *ServerReconciler) reconcileServer(ctx context.Context, server *apiv1.Se
if deploy.Status.ReadyReplicas == 0 {
server.Status.Ready = false
meta.SetStatusCondition(&server.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionDeployed,
+ Type: apiv1.ConditionServing,
Status: metav1.ConditionFalse,
Reason: apiv1.ReasonDeploymentNotReady,
ObservedGeneration: server.Generation,
@@ -289,7 +288,7 @@ func (r *ServerReconciler) reconcileServer(ctx context.Context, server *apiv1.Se
} else {
server.Status.Ready = true
meta.SetStatusCondition(&server.Status.Conditions, metav1.Condition{
- Type: apiv1.ConditionDeployed,
+ Type: apiv1.ConditionServing,
Status: metav1.ConditionTrue,
Reason: apiv1.ReasonDeploymentReady,
ObservedGeneration: server.Generation,
@@ -336,7 +335,7 @@ func (r *ServerReconciler) serverService(server *apiv1.Server, model *apiv1.Mode
}
func withServerSelector(server *apiv1.Server, labels map[string]string) map[string]string {
- labels["component"] = "server"
+ labels["role"] = "run"
labels["server"] = server.Name
return labels
}
diff --git a/internal/controller/utils.go b/internal/controller/utils.go
index 0b3fae65..9c694fef 100644
--- a/internal/controller/utils.go
+++ b/internal/controller/utils.go
@@ -17,9 +17,10 @@ import (
type result struct {
ctrl.Result
success bool
+ failure bool
}
-func reconcileJob(ctx context.Context, c client.Client, job *batchv1.Job, condition string) (result, error) {
+func reconcileJob(ctx context.Context, c client.Client, job *batchv1.Job) (result, error) {
if err := c.Create(ctx, job); client.IgnoreAlreadyExists(err) != nil {
return result{}, fmt.Errorf("creating Job: %w", err)
}
@@ -27,12 +28,24 @@ func reconcileJob(ctx context.Context, c client.Client, job *batchv1.Job, condit
if err := c.Get(ctx, client.ObjectKeyFromObject(job), job); err != nil {
return result{}, fmt.Errorf("geting Job: %w", err)
}
- if job.Status.Succeeded < 1 {
- // Allow Job watch to requeue.
- return result{}, nil
- }
- return result{success: true}, nil
+ complete, failed := jobResult(job)
+
+ return result{success: complete, failure: failed}, nil
+}
+
+func jobResult(job *batchv1.Job) (complete bool, failed bool) {
+ for _, c := range job.Status.Conditions {
+ if c.Type == batchv1.JobComplete && c.Status == corev1.ConditionTrue {
+ complete = true
+ return
+ }
+ if c.Type == batchv1.JobFailed && c.Status == corev1.ConditionTrue {
+ failed = true
+ return
+ }
+ }
+ return
}
func isPodReady(pod *corev1.Pod) bool {
diff --git a/internal/tui/common.go b/internal/tui/common.go
new file mode 100644
index 00000000..141a637c
--- /dev/null
+++ b/internal/tui/common.go
@@ -0,0 +1,304 @@
+package tui
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+ "strconv"
+ "strings"
+
+ tea "github.com/charmbracelet/bubbletea"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+ "k8s.io/apimachinery/pkg/types"
+
+ apiv1 "github.com/substratusai/substratus/api/v1"
+ "github.com/substratusai/substratus/internal/cli/utils"
+ "github.com/substratusai/substratus/internal/client"
+)
+
+var (
+ P *tea.Program
+ LogFile *os.File
+)
+
+func init() {
+ // Log to a file. Useful in debugging since you can't really log to stdout.
+ var err error
+ LogFile, err = tea.LogToFile("/tmp/sub.log", "")
+ if err != nil {
+ panic(err)
+ }
+}
+
+type Namespace struct {
+ Specified string
+ Contextual string
+}
+
+func (n Namespace) Set(obj client.Object) {
+ if n.Specified != "" {
+ obj.SetNamespace(n.Specified)
+ } else if obj.GetNamespace() == "" {
+ ns := "default"
+ if n.Contextual != "" {
+ ns = n.Contextual
+ }
+ obj.SetNamespace(ns)
+ }
+}
+
+type status int
+
+const (
+ notStarted = status(0)
+ inProgress = status(1)
+ completed = status(2)
+)
+
+type localURLMsg string
+
+type (
+ tarballCompleteMsg *client.Tarball
+ fileTarredMsg string
+)
+
+func prepareTarballCmd(ctx context.Context, dir string) tea.Cmd {
+ return func() tea.Msg {
+ log.Println("Preparing tarball")
+ tarball, err := client.PrepareImageTarball(ctx, dir, func(file string) {
+ log.Println("tarred", file)
+ P.Send(fileTarredMsg(file))
+ })
+ if err != nil {
+ log.Println("Error", err)
+ return fmt.Errorf("preparing tarball: %w", err)
+ }
+ return tarballCompleteMsg(tarball)
+ }
+}
+
+type (
+ tarballUploadedMsg struct {
+ client.Object
+ }
+ uploadTarballProgressMsg float64
+)
+
+func uploadTarballCmd(ctx context.Context, res *client.Resource, obj client.Object, tarball *client.Tarball) tea.Cmd {
+ return func() tea.Msg {
+ log.Println("Uploading tarball")
+ err := res.Upload(ctx, obj, tarball, func(percentage float64) {
+ log.Printf("Upload percentage: %v", percentage)
+ P.Send(uploadTarballProgressMsg(percentage))
+ })
+ if err != nil {
+ log.Println("Upload failed", err)
+ return fmt.Errorf("uploading: %w", err)
+ }
+ log.Println("Upload completed")
+ return tarballUploadedMsg{Object: obj}
+ }
+}
+
+func specifyUpload(obj client.Object, tarball *client.Tarball) error {
+ if err := client.ClearImage(obj); err != nil {
+ return fmt.Errorf("clearing image in spec: %w", err)
+ }
+ if err := client.SetUploadContainerSpec(obj, tarball, utils.NewUUID()); err != nil {
+ return fmt.Errorf("setting upload in spec: %w", err)
+ }
+ return nil
+}
+
+type appliedWithUploadMsg struct {
+ client.Object
+}
+
+func applyWithUploadCmd(ctx context.Context, res *client.Resource, obj client.Object, tarball *client.Tarball) tea.Cmd {
+ return func() tea.Msg {
+ if err := specifyUpload(obj, tarball); err != nil {
+ return fmt.Errorf("specifying upload: %w", err)
+ }
+ if err := res.Apply(obj, true); err != nil {
+ return fmt.Errorf("applying: %w", err)
+ }
+ return appliedWithUploadMsg{Object: obj}
+ }
+}
+
+type appliedMsg struct {
+ client.Object
+}
+
+func applyCmd(ctx context.Context, res *client.Resource, obj client.Object) tea.Cmd {
+ return func() tea.Msg {
+ if err := res.Apply(obj, true); err != nil {
+ return fmt.Errorf("applying: %w", err)
+ }
+ return appliedMsg{Object: obj}
+ }
+}
+
+type createdWithUploadMsg struct {
+ client.Object
+}
+
+func createWithUploadCmd(ctx context.Context, res *client.Resource, obj client.Object, tarball *client.Tarball) tea.Cmd {
+ return func() tea.Msg {
+ if err := specifyUpload(obj, tarball); err != nil {
+ return fmt.Errorf("specifying upload: %w", err)
+ }
+
+ lowerKind := strings.ToLower(obj.GetObjectKind().GroupVersionKind().Kind)
+ if obj.GetLabels() == nil {
+ obj.SetLabels(map[string]string{})
+ }
+ obj.GetLabels()[lowerKind] = obj.GetName()
+
+ list, err := res.List(obj.GetNamespace(), obj.GetObjectKind().GroupVersionKind().Version, &metav1.ListOptions{
+ LabelSelector: labels.SelectorFromSet(map[string]string{
+ lowerKind: obj.GetName(),
+ }).String(),
+ })
+ if err != nil {
+ return fmt.Errorf("listing: %w", err)
+ }
+
+ var version int
+ switch list := list.(type) {
+ case *apiv1.ModelList:
+ version, err = nextModelVersion(list)
+ if err != nil {
+ return fmt.Errorf("next model version: %w", err)
+ }
+ case *apiv1.DatasetList:
+ version, err = nextDatasetVersion(list)
+ if err != nil {
+ return fmt.Errorf("next dataset version: %w", err)
+ }
+ default:
+ return fmt.Errorf("unrecognized list type: %T", list)
+ }
+
+ log.Printf("Next version: %v", version)
+
+ obj.SetName(fmt.Sprintf("%v.v%v", obj.GetName(), version))
+ obj.GetLabels()["version"] = fmt.Sprintf("%v", version)
+ if _, err := res.Create(obj.GetNamespace(), true, obj); err != nil {
+ return fmt.Errorf("creating: %w", err)
+ }
+ return createdWithUploadMsg{Object: obj}
+ }
+}
+
+type objectReadyMsg struct {
+ client.Object
+}
+
+type objectUpdateMsg struct {
+ client.Object
+}
+
+func waitReadyCmd(ctx context.Context, res *client.Resource, obj client.Object) tea.Cmd {
+ return func() tea.Msg {
+ if err := res.WaitReady(ctx, obj, func(updatedObj client.Object) {
+ P.Send(objectUpdateMsg{Object: updatedObj})
+ }); err != nil {
+ return fmt.Errorf("waiting to be ready: %w", err)
+ }
+ return objectReadyMsg{Object: obj}
+ }
+}
+
+func nextModelVersion(list *apiv1.ModelList) (int, error) {
+ var highestVersion int
+ for _, item := range list.Items {
+ v, err := strconv.Atoi(item.GetLabels()["version"])
+ if err != nil {
+ return 0, fmt.Errorf("version label to int: %w", err)
+ }
+ if v > highestVersion {
+ highestVersion = v
+ }
+ }
+
+ return highestVersion + 1, nil
+}
+
+func nextDatasetVersion(list *apiv1.DatasetList) (int, error) {
+ var highestVersion int
+ for _, item := range list.Items {
+ v, err := strconv.Atoi(item.GetLabels()["version"])
+ if err != nil {
+ return 0, fmt.Errorf("version label to int: %w", err)
+ }
+ if v > highestVersion {
+ highestVersion = v
+ }
+ }
+
+ return highestVersion + 1, nil
+}
+
+type suspendedMsg struct {
+ error error
+}
+
+func suspendCmd(ctx context.Context, res *client.Resource, obj client.Object) tea.Cmd {
+ return func() tea.Msg {
+ log.Println("Suspending")
+ _, err := res.Patch(obj.GetNamespace(), obj.GetName(), types.MergePatchType, []byte(`{"spec": {"suspend": true} }`), &metav1.PatchOptions{})
+ if err != nil {
+ log.Printf("Error suspending: %v", err)
+ return suspendedMsg{error: err}
+ }
+ return suspendedMsg{}
+ }
+}
+
+type deletedMsg struct {
+ name string
+ error error
+}
+
+func deleteCmd(ctx context.Context, res *client.Resource, obj client.Object) tea.Cmd {
+ return func() tea.Msg {
+ name := obj.GetName()
+
+ log.Println("Deleting")
+ _, err := res.Delete(obj.GetNamespace(), obj.GetName())
+ if err != nil {
+ log.Printf("Error deleting: %v", err)
+ return deletedMsg{name: name, error: err}
+ }
+
+ return deletedMsg{name: name}
+ }
+}
+
+type readManifestMsg struct {
+ obj client.Object
+}
+
+func readManifest(path string) tea.Cmd {
+ return func() tea.Msg {
+ log.Println("Reading manifest")
+
+ manifest, err := os.ReadFile(path)
+ if err != nil {
+ return fmt.Errorf("reading file: %w", err)
+ }
+
+ var obj client.Object
+ obj, err = client.Decode(manifest)
+ if err != nil {
+ return fmt.Errorf("decoding: %w", err)
+ }
+
+ return readManifestMsg{
+ obj: obj,
+ }
+ }
+}
diff --git a/internal/tui/delete.go b/internal/tui/delete.go
new file mode 100644
index 00000000..a780c6ec
--- /dev/null
+++ b/internal/tui/delete.go
@@ -0,0 +1,162 @@
+package tui
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/charmbracelet/lipgloss"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "github.com/substratusai/substratus/internal/client"
+)
+
+type DeleteModel struct {
+ // Cancellation
+ Ctx context.Context
+
+ // Config
+ Scope string
+ Namespace Namespace
+
+ // Clients
+ Client client.Interface
+
+ resource *client.Resource
+ toDelete []client.Object
+ deleted []string
+
+ Style lipgloss.Style
+
+ // End times
+ goodbye string
+ finalError error
+}
+
+func (m *DeleteModel) New() DeleteModel {
+ m.Style = appStyle
+ if m.toDelete == nil {
+ m.toDelete = []client.Object{}
+ }
+ return *m
+}
+
+type deleteInitMsg struct{}
+
+func (m DeleteModel) Init() tea.Cmd {
+ //if len(m.Objects) == 0 {
+ // return listCmd(m.Ctx, m.Resource, m.Scope)
+ //} else {
+ // cmds := make([]tea.Cmd, 0, len(m.Objects))
+ // for _, obj := range m.Objects {
+ // cmds = append(cmds, deleteCmd(m.Ctx, m.Resource, obj))
+ // }
+ // return tea.Batch(cmds...)
+ //}
+
+ return func() tea.Msg { return deleteInitMsg{} }
+}
+
+func (m DeleteModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+ switch msg := msg.(type) {
+
+ case deleteInitMsg:
+ log.Println("running init")
+ obj, err := scopeToObject(m.Scope)
+ if err != nil {
+ m.finalError = fmt.Errorf("scope to object: %w", err)
+ return m, nil
+ }
+ m.Namespace.Set(obj)
+
+ res, err := m.Client.Resource(obj)
+ if err != nil {
+ m.finalError = fmt.Errorf("resource client: %w", err)
+ return m, nil
+ }
+ m.resource = res
+
+ return m, getDeletionTargetsCmd(m.Ctx, m.resource, obj)
+
+ case tea.KeyMsg:
+ log.Println("Received key msg:", msg.String())
+ if msg.String() == "q" {
+ return m, tea.Quit
+ }
+
+ case deletionListMsg:
+ var cmds []tea.Cmd
+ m.toDelete = msg.items
+ for _, obj := range msg.items {
+ log.Printf("to-delete: %v", obj.GetName())
+ // TODO: Implement a confirmation flow.
+ cmds = append(cmds, deleteCmd(m.Ctx, m.resource, obj))
+ }
+ return m, tea.Sequence(cmds...)
+
+ case deletedMsg:
+ if msg.error != nil {
+ m.finalError = msg.error
+ } else {
+ m.deleted = append(m.deleted, msg.name)
+ }
+ if len(m.deleted) == len(m.toDelete) {
+ return m, tea.Quit
+ }
+
+ case tea.WindowSizeMsg:
+ m.Style.Width(msg.Width)
+ }
+
+ return m, nil
+}
+
+// View returns a string based on data in the model. That string which will be
+// rendered to the terminal.
+func (m DeleteModel) View() (v string) {
+ defer func() {
+ v += helpStyle("Press \"q\" to quit")
+ v = m.Style.Render(v)
+ }()
+
+ if m.finalError != nil {
+ v += errorStyle.Width(m.Style.GetWidth()-m.Style.GetHorizontalMargins()-10).Render("Error: "+m.finalError.Error()) + "\n"
+ return
+ }
+
+ if m.goodbye != "" {
+ v += m.goodbye + "\n"
+ return
+ }
+
+ for _, name := range m.deleted {
+ v += checkMark.String() + " " + name + ": deleted\n"
+ }
+
+ return
+}
+
+type deletionListMsg struct {
+ items []client.Object
+}
+
+func getDeletionTargetsCmd(ctx context.Context, res *client.Resource, obj client.Object) tea.Cmd {
+ log.Printf("getting deletion targets: %v/%v", obj.GetNamespace(), obj.GetName())
+ return func() tea.Msg {
+ if obj.GetName() != "" {
+ fetched, err := res.Get(obj.GetNamespace(), obj.GetName())
+ if err != nil {
+ return fmt.Errorf("get: %w", err)
+ }
+ return deletionListMsg{items: []client.Object{fetched.(client.Object)}}
+ } else {
+ items, err := res.List(obj.GetNamespace(), "", &metav1.ListOptions{})
+ if err != nil {
+ return fmt.Errorf("list: %w", err)
+ }
+ log.Printf("%T", items)
+ panic("NOT IMPLEMENTED")
+ }
+ }
+}
diff --git a/internal/tui/get.go b/internal/tui/get.go
new file mode 100644
index 00000000..de3f44bb
--- /dev/null
+++ b/internal/tui/get.go
@@ -0,0 +1,369 @@
+package tui
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "math"
+ "slices"
+ "sort"
+ "strings"
+
+ "github.com/charmbracelet/bubbles/spinner"
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/charmbracelet/lipgloss"
+ "k8s.io/apimachinery/pkg/api/meta"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/watch"
+
+ apiv1 "github.com/substratusai/substratus/api/v1"
+ "github.com/substratusai/substratus/internal/client"
+)
+
+type GetModel struct {
+ // Cancellation
+ Ctx context.Context
+
+ // Config
+ Scope string
+ Namespace string
+
+ // Clients
+ Client client.Interface
+
+ // End times
+ finalError error
+
+ objects map[string]map[string]listedObject
+
+ Style lipgloss.Style
+}
+
+type listedObject struct {
+ object
+ spinner spinner.Model
+}
+
+func newGetObjectMap() map[string]map[string]listedObject {
+ return map[string]map[string]listedObject{
+ "notebooks": {},
+ "datasets": {},
+ "models": {},
+ "servers": {},
+ }
+}
+
+func (m *GetModel) New() GetModel {
+ m.objects = newGetObjectMap()
+ m.Style = appStyle
+ return *m
+}
+
+func (m GetModel) Init() tea.Cmd {
+ return watchCmd(m.Ctx, m.Client, m.Namespace, m.Scope)
+}
+
+func (m GetModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+ switch msg := msg.(type) {
+ case tea.KeyMsg:
+ log.Println("Received key msg:", msg.String())
+ if msg.String() == "q" {
+ return m, tea.Quit
+ }
+
+ case watchMsg:
+ var cmd tea.Cmd
+ switch msg.Type {
+ case watch.Deleted:
+ delete(m.objects[msg.resource], msg.Object.(object).GetName())
+ case watch.Error:
+ log.Printf("Watch error: %v", msg.Object)
+ default:
+ o := msg.Object.(client.Object)
+ name := o.GetName()
+ log.Printf("Watch event: %v: %v", msg.resource, name)
+
+ lo := m.objects[msg.resource][name]
+ lo.object = msg.Object.(object)
+ if msg.Type == watch.Added {
+ lo.spinner = spinner.New(spinner.WithSpinner(spinner.MiniDot), spinner.WithStyle(activeSpinnerStyle))
+ cmd = lo.spinner.Tick
+ }
+ m.objects[msg.resource][name] = lo
+ }
+ return m, cmd
+
+ case spinner.TickMsg:
+ for kind := range m.objects {
+ for name := range m.objects[kind] {
+ o := m.objects[kind][name]
+ var cmd tea.Cmd
+ if o.spinner.ID() == msg.ID {
+ o.spinner, cmd = o.spinner.Update(msg)
+ m.objects[kind][name] = o
+ return m, cmd
+ }
+ }
+ }
+ return m, nil
+
+ case tea.WindowSizeMsg:
+ m.Style.Width(msg.Width)
+
+ case error:
+ m.finalError = msg
+ return m, nil
+ }
+
+ return m, nil
+}
+
+// View returns a string based on data in the model. That string which will be
+// rendered to the terminal.
+func (m GetModel) View() (v string) {
+ defer func() {
+ v = m.Style.Render(v)
+ }()
+
+ if m.finalError != nil {
+ v += errorStyle.Render("Error: "+m.finalError.Error()) + "\n"
+ v += helpStyle("Press \"q\" to quit")
+ return v
+ }
+
+ scopeResource, scopeName := splitScope(m.Scope)
+
+ var total int
+ for _, resource := range []struct {
+ plural string
+ versioned bool
+ }{
+ {plural: "notebooks", versioned: false},
+ {plural: "datasets", versioned: true},
+ {plural: "models", versioned: true},
+ {plural: "servers", versioned: false},
+ } {
+ if len(m.objects[resource.plural]) == 0 {
+ continue
+ }
+
+ if scopeResource == "" {
+ v += resource.plural + "/" + "\n"
+ }
+
+ var names []string
+ for name := range m.objects[resource.plural] {
+ names = append(names, name)
+ total++
+ }
+ sort.Strings(names)
+
+ if !resource.versioned {
+ for _, name := range names {
+ o := m.objects[resource.plural][name]
+
+ var indicator string
+ if o.GetStatusReady() {
+ indicator = checkMark.String()
+ } else {
+ indicator = o.spinner.View()
+ }
+ v += "" + indicator + " " + name + "\n"
+ }
+ } else {
+ type objectVersions struct {
+ unversionedName string
+ versions []listedObject
+ }
+
+ var groups []objectVersions
+
+ var lastUnversionedName string
+ // var longestName int
+ const longestName = 30
+ for _, name := range names {
+ o := m.objects[resource.plural][name]
+ lowerKind := strings.TrimSuffix(resource.plural, "s")
+ unversionedName := o.GetLabels()[lowerKind]
+
+ if unversionedName != lastUnversionedName {
+ groups = append(groups, objectVersions{
+ unversionedName: unversionedName,
+ versions: []listedObject{o},
+ })
+ } else {
+ groups[len(groups)-1].versions = append(groups[len(groups)-1].versions, o)
+ }
+
+ lastUnversionedName = unversionedName
+ //if n := len(name); n > longestName {
+ // longestName = n + 6
+ //}
+ }
+
+ for gi, g := range groups {
+ type versionDisplay struct {
+ indicator string
+ version string
+ }
+ var displayVersions []versionDisplay
+ for _, o := range g.versions {
+ version := o.GetLabels()["version"]
+
+ var indicator string
+ if o.GetStatusReady() {
+ indicator = checkMark.String()
+ } else if c := meta.FindStatusCondition(*o.GetConditions(), apiv1.ConditionComplete); c != nil && c.Reason == apiv1.ReasonJobFailed {
+ indicator = xMark.String()
+ } else {
+ indicator = o.spinner.View()
+ }
+ displayVersions = append(displayVersions, versionDisplay{
+ indicator: indicator,
+ version: version,
+ })
+ }
+
+ // Latest first
+ slices.Reverse(displayVersions)
+
+ var otherVersions []string
+ for _, other := range displayVersions[1:] {
+ otherVersions = append(otherVersions, fmt.Sprintf("%v.v%v", other.indicator, other.version))
+ }
+
+ primary := displayVersions[0].indicator + " " +
+ g.unversionedName + ".v" +
+ displayVersions[0].version
+
+ verWidth := int(math.Min(float64(60), float64(m.Style.GetWidth()-m.Style.GetHorizontalMargins()-longestName-18)))
+ v += lipgloss.JoinHorizontal(
+ lipgloss.Top,
+ lipgloss.NewStyle().Width(longestName).MarginLeft(0).MarginRight(2).Align(lipgloss.Left).Render(primary),
+ lipgloss.NewStyle().Width(verWidth).MarginRight(4).Align(lipgloss.Right).Render(strings.Join(otherVersions, " ")),
+ )
+ if gi < len(groups) {
+ v += "\n"
+ }
+ }
+
+ }
+ v += "\n"
+ }
+
+ if scopeName == "" {
+ v += fmt.Sprintf("\nTotal: %v\n", total)
+ }
+
+ v += helpStyle("Press \"q\" to quit")
+
+ return v
+}
+
+type watchMsg struct {
+ watch.Event
+ resource string
+}
+
+type object interface {
+ client.Object
+ GetConditions() *[]metav1.Condition
+ GetStatusReady() bool
+}
+
+func watchCmd(ctx context.Context, c client.Interface, namespace, scope string) tea.Cmd {
+ pluralName := func(s string) string {
+ return strings.ToLower(s) + "s"
+ }
+
+ return func() tea.Msg {
+ log.Println("Starting watch")
+
+ objs, err := scopeToObjects(scope)
+ if err != nil {
+ return fmt.Errorf("parsing search term: %v", err)
+ }
+
+ for _, obj := range objs {
+ res, err := c.Resource(obj)
+ if err != nil {
+ return fmt.Errorf("resource client: %w", err)
+ }
+
+ kind := obj.GetObjectKind().GroupVersionKind().Kind
+ log.Printf("Starting watch: %v", kind)
+
+ w, err := res.Watch(ctx, namespace, obj, &metav1.ListOptions{})
+ if err != nil {
+ return fmt.Errorf("watch: %w", err)
+ }
+ go func() {
+ for event := range w.ResultChan() {
+ P.Send(watchMsg{Event: event, resource: pluralName(kind)})
+ }
+ }()
+ }
+
+ return nil
+ }
+}
+
+// scopeToObjects maps a scope string to a slice of objects.
+// "" --> All Substratus kinds
+// "models" --> All Models
+// "models/" --> Single Model
+func scopeToObjects(scope string) ([]client.Object, error) {
+ if scope == "" {
+ return []client.Object{
+ &apiv1.Notebook{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Notebook"}},
+ &apiv1.Dataset{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Dataset"}},
+ &apiv1.Model{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Model"}},
+ &apiv1.Server{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Server"}},
+ }, nil
+ }
+
+ singleObj, err := scopeToObject(scope)
+ if err != nil {
+ return nil, err
+ }
+
+ return []client.Object{singleObj}, nil
+}
+
+func scopeToObject(scope string) (client.Object, error) {
+ res, name := splitScope(scope)
+ if res == "" && name == "" {
+ return nil, fmt.Errorf("Invalid scope: %v", scope)
+ }
+
+ var obj client.Object
+ switch res {
+ case "notebooks":
+ obj = &apiv1.Notebook{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Notebook"}}
+ case "datasets":
+ obj = &apiv1.Dataset{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Dataset"}}
+ case "models":
+ obj = &apiv1.Model{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Model"}}
+ case "servers":
+ obj = &apiv1.Server{TypeMeta: metav1.TypeMeta{APIVersion: "substratus.ai/v1", Kind: "Server"}}
+ default:
+ return nil, fmt.Errorf("Invalid scope: %v", scope)
+ }
+
+ if name != "" {
+ obj.SetName(name)
+ }
+
+ return obj, nil
+}
+
+func splitScope(scope string) (string, string) {
+ split := strings.Split(scope, "/")
+ if len(split) == 1 {
+ return split[0], ""
+ }
+ if len(split) == 2 {
+ return split[0], split[1]
+ }
+ return "", ""
+}
diff --git a/internal/tui/infer_chat.go b/internal/tui/infer_chat.go
new file mode 100644
index 00000000..42d3781e
--- /dev/null
+++ b/internal/tui/infer_chat.go
@@ -0,0 +1,92 @@
+package tui
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/charmbracelet/bubbles/textarea"
+ "github.com/charmbracelet/bubbles/viewport"
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/charmbracelet/lipgloss"
+)
+
+type (
+ errMsg error
+)
+
+type ChatModel struct {
+ viewport viewport.Model
+ messages []string
+ textarea textarea.Model
+ senderStyle lipgloss.Style
+ err error
+}
+
+func (m ChatModel) Init() tea.Cmd {
+ ta := textarea.New()
+ ta.Placeholder = "Send a message..."
+ ta.Focus()
+
+ ta.Prompt = "┃ "
+ ta.CharLimit = 280
+
+ ta.SetWidth(30)
+ ta.SetHeight(3)
+
+ // Remove cursor line styling
+ ta.FocusedStyle.CursorLine = lipgloss.NewStyle()
+
+ ta.ShowLineNumbers = false
+
+ vp := viewport.New(30, 5)
+ vp.SetContent(`Welcome to the chat room!
+Type a message and press Enter to send.`)
+
+ ta.KeyMap.InsertNewline.SetEnabled(false)
+
+ m.textarea = ta
+ m.messages = []string{}
+ m.viewport = vp
+ m.senderStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("5"))
+
+ return textarea.Blink
+}
+
+func (m ChatModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+ var (
+ tiCmd tea.Cmd
+ vpCmd tea.Cmd
+ )
+
+ m.textarea, tiCmd = m.textarea.Update(msg)
+ m.viewport, vpCmd = m.viewport.Update(msg)
+
+ switch msg := msg.(type) {
+ case tea.KeyMsg:
+ switch msg.Type {
+ case tea.KeyCtrlC, tea.KeyEsc:
+ fmt.Println(m.textarea.Value())
+ return m, tea.Quit
+ case tea.KeyEnter:
+ m.messages = append(m.messages, m.senderStyle.Render("You: ")+m.textarea.Value())
+ m.viewport.SetContent(strings.Join(m.messages, "\n"))
+ m.textarea.Reset()
+ m.viewport.GotoBottom()
+ }
+
+ // We handle errors just like any other message
+ case errMsg:
+ m.err = msg
+ return m, nil
+ }
+
+ return m, tea.Batch(tiCmd, vpCmd)
+}
+
+func (m ChatModel) View() string {
+ return fmt.Sprintf(
+ "%s\n\n%s",
+ m.viewport.View(),
+ m.textarea.View(),
+ ) + "\n\n"
+}
diff --git a/internal/tui/manifests.go b/internal/tui/manifests.go
new file mode 100644
index 00000000..c7097aa8
--- /dev/null
+++ b/internal/tui/manifests.go
@@ -0,0 +1,187 @@
+package tui
+
+import (
+ "bytes"
+ "fmt"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/charmbracelet/lipgloss"
+
+ apiv1 "github.com/substratusai/substratus/api/v1"
+ "github.com/substratusai/substratus/internal/client"
+)
+
+type manifestsModel struct {
+ Path string
+ Filename string
+
+ // Kinds is a list of manifest kinds to include in results,
+ // ordered by preference.
+ Kinds []string
+
+ reading status
+
+ Style lipgloss.Style
+}
+
+// New initializes all internal fields.
+func (m *manifestsModel) New() manifestsModel {
+ return *m
+}
+
+func (m manifestsModel) Active() bool {
+ return m.reading == inProgress
+}
+
+func (m manifestsModel) Init() tea.Cmd {
+ return tea.Sequence(
+ func() tea.Msg { return manifestsInitMsg{} },
+ findSubstratusManifests(m.Path, m.Filename),
+ )
+}
+
+type manifestsInitMsg struct{}
+
+func (m manifestsModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+ switch msg := msg.(type) {
+ case manifestsInitMsg:
+ log.Println("Initializing manifests")
+ m.reading = inProgress
+
+ case readManifestMsg:
+ m.reading = completed
+
+ return m, func() tea.Msg { return manifestSelectedMsg{obj: msg.obj} }
+
+ case substratusManifestsMsg:
+ m.reading = completed
+
+ var n int
+ var single client.Object
+ for _, k := range m.Kinds {
+ items := msg.manifests[k]
+ if single == nil && len(items) > 0 {
+ single = items[0]
+ }
+ n += len(items)
+ }
+
+ log.Printf("Found (filtered) manifests: %v", n)
+
+ if n == 0 {
+ return m, func() tea.Msg { return fmt.Errorf("No substratus Server kinds found in *.yaml") }
+ } else if n == 1 {
+ return m, func() tea.Msg { return manifestSelectedMsg{obj: single} }
+ } else {
+ // TODO: Selector
+ return m, func() tea.Msg { return manifestSelectedMsg{obj: single} }
+ }
+
+ }
+
+ return m, nil
+}
+
+// View returns a string based on data in the model. That string which will be
+// rendered to the terminal.
+func (m manifestsModel) View() (v string) {
+ defer func() {
+ if v != "" {
+ v = m.Style.Render(v)
+ }
+ }()
+
+ if m.reading == inProgress {
+ v += "Reading manifests..."
+ }
+
+ return
+}
+
+type manifestSelectedMsg struct {
+ obj client.Object
+}
+
+type substratusManifestsMsg struct {
+ manifests map[string][]client.Object
+}
+
+func findSubstratusManifests(path, filename string) tea.Cmd {
+ return func() tea.Msg {
+ msg := substratusManifestsMsg{
+ manifests: map[string][]client.Object{},
+ }
+
+ var fp string
+ if filename != "" {
+ fp = filepath.Join(path, filename)
+ manifest, err := os.ReadFile(fp)
+ if err != nil {
+ return fmt.Errorf("reading file: %w", err)
+ }
+ if err := manifestToObjects(manifest, msg.manifests); err != nil {
+ return fmt.Errorf("reading manifests in file: %v: %w", filename, err)
+ }
+ } else {
+ if path == "" {
+ var err error
+ path, err = os.Getwd()
+ if err != nil {
+ return err
+ }
+ }
+
+ fp = filepath.Join(path, "*.yaml")
+ matches, err := filepath.Glob(fp)
+ if err != nil {
+ return err
+ }
+ for _, p := range matches {
+ manifest, err := os.ReadFile(p)
+ if err != nil {
+ return fmt.Errorf("reading file: %w", err)
+ }
+
+ if err := manifestToObjects(manifest, msg.manifests); err != nil {
+ return fmt.Errorf("reading manifests in file: %v: %w", p, err)
+ }
+ }
+ }
+ if len(msg.manifests) == 0 {
+ return fmt.Errorf("No manifests found: %v", fp)
+ }
+ return msg
+ }
+}
+
+func manifestToObjects(manifest []byte, m map[string][]client.Object) error {
+ split := bytes.Split(manifest, []byte("---\n"))
+ for _, doc := range split {
+ if strings.TrimSpace(string(doc)) == "" {
+ continue
+ }
+
+ obj, err := client.Decode(doc)
+ if err != nil {
+ return fmt.Errorf("decoding: %w", err)
+ }
+ if obj == nil {
+ return nil
+ }
+
+ switch t := obj.(type) {
+ case *apiv1.Model, *apiv1.Dataset, *apiv1.Server, *apiv1.Notebook:
+ kind := t.GetObjectKind().GroupVersionKind().Kind
+ if m[kind] == nil {
+ m[kind] = make([]client.Object, 0)
+ }
+ m[kind] = append(m[kind], obj)
+ }
+ }
+
+ return nil
+}
diff --git a/internal/tui/notebook.go b/internal/tui/notebook.go
new file mode 100644
index 00000000..ea681881
--- /dev/null
+++ b/internal/tui/notebook.go
@@ -0,0 +1,330 @@
+package tui
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log"
+
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/charmbracelet/lipgloss"
+ "github.com/pkg/browser"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/utils/ptr"
+
+ apiv1 "github.com/substratusai/substratus/api/v1"
+ "github.com/substratusai/substratus/internal/client"
+)
+
+type NotebookModel struct {
+ // Cancellation
+ Ctx context.Context
+
+ // Config
+ Path string
+ Filename string
+ Namespace Namespace
+ NoOpenBrowser bool
+
+ // Clients
+ Client client.Interface
+ K8s *kubernetes.Clientset
+
+ // Current notebook
+ notebook *apiv1.Notebook
+ resource *client.Resource
+
+ // Proceses
+ manifests manifestsModel
+ upload uploadModel
+ readiness readinessModel
+ pods podsModel
+
+ // File syncing
+ syncingFiles status
+ currentSyncingFile string
+ lastSyncFailure error
+
+ // Ready to open browser
+ portForwarding status
+ localURL string
+
+ // End times
+ quitting bool
+ goodbye string
+ finalError error
+
+ Style lipgloss.Style
+}
+
+func (m NotebookModel) cleanupAndQuitCmd() tea.Msg {
+ m.upload.cleanup()
+ return tea.Quit()
+}
+
+func (m *NotebookModel) New() NotebookModel {
+ m.manifests = (&manifestsModel{
+ Path: m.Path,
+ Filename: m.Filename,
+ Kinds: []string{"Notebook", "Model", "Dataset"},
+ }).New()
+ m.upload = (&uploadModel{
+ Ctx: m.Ctx,
+ Client: m.Client,
+ Path: m.Path,
+ Mode: uploadModeApply,
+ }).New()
+ m.readiness = (&readinessModel{
+ Ctx: m.Ctx,
+ Client: m.Client,
+ }).New()
+ m.pods = (&podsModel{
+ Ctx: m.Ctx,
+ Client: m.Client,
+ K8s: m.K8s,
+ }).New()
+
+ m.Style = appStyle
+
+ return *m
+}
+
+func (m NotebookModel) Init() tea.Cmd {
+ // return readManifest(filepath.Join(m.Path, m.Filename))
+ return m.manifests.Init()
+}
+
+func (m NotebookModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+ var cmds []tea.Cmd
+
+ log.Printf("MSG: %T", msg)
+
+ {
+ mdl, cmd := m.manifests.Update(msg)
+ m.manifests = mdl.(manifestsModel)
+ cmds = append(cmds, cmd)
+ }
+
+ {
+ mdl, cmd := m.upload.Update(msg)
+ m.upload = mdl.(uploadModel)
+ cmds = append(cmds, cmd)
+ }
+
+ {
+ mdl, cmd := m.readiness.Update(msg)
+ m.readiness = mdl.(readinessModel)
+ cmds = append(cmds, cmd)
+ }
+
+ {
+ mdl, cmd := m.pods.Update(msg)
+ m.pods = mdl.(podsModel)
+ cmds = append(cmds, cmd)
+ }
+
+ switch msg := msg.(type) {
+ case manifestSelectedMsg:
+ m.Namespace.Set(msg.obj)
+ nb, err := client.NotebookForObject(msg.obj)
+ if err != nil {
+ m.finalError = fmt.Errorf("determining notebook: %w", err)
+ }
+ nb.Spec.Suspend = ptr.To(false)
+ m.notebook = nb
+
+ res, err := m.Client.Resource(m.notebook)
+ if err != nil {
+ m.finalError = fmt.Errorf("resource client: %w", err)
+ }
+ m.resource = res
+
+ m.upload.Object = m.notebook
+ m.upload.Resource = m.resource
+ cmds = append(cmds, m.upload.Init())
+
+ case tea.KeyMsg:
+ log.Println("Received key msg:", msg.String())
+ if m.quitting {
+ switch msg.String() {
+ case "esc":
+ if m.finalError == nil {
+ m.quitting = false
+ }
+
+ // "Leave be" results in issues where a build will eventually replace the Notebook
+ // and the command will error out due to a failure on the previous notebook nbwatch
+ // command... revisit later.
+ //
+ // case "l":
+ // cmds = append(cmds, m.cleanupAndQuitCmd)
+ case "s":
+ cmds = append(cmds, suspendCmd(context.Background(), m.resource, m.notebook))
+ case "d":
+ cmds = append(cmds, deleteCmd(context.Background(), m.resource, m.notebook))
+ }
+ } else {
+ if msg.String() == "q" {
+ m.quitting = true
+ }
+ }
+
+ case suspendedMsg:
+ if msg.error != nil {
+ m.finalError = msg.error
+ } else {
+ m.goodbye = "Notebook suspended."
+ }
+ cmds = append(cmds, m.cleanupAndQuitCmd)
+
+ case deletedMsg:
+ if msg.error != nil {
+ m.finalError = msg.error
+ } else {
+ m.goodbye = "Notebook deleted."
+ }
+ cmds = append(cmds, m.cleanupAndQuitCmd)
+
+ case tarballUploadedMsg:
+ m.notebook = msg.Object.(*apiv1.Notebook)
+
+ m.readiness.Object = m.notebook
+ m.readiness.Resource = m.resource
+ m.pods.Object = m.notebook
+ m.pods.Resource = m.resource
+ cmds = append(cmds,
+ m.readiness.Init(),
+ m.pods.Init(),
+ )
+
+ case objectReadyMsg:
+ m.notebook = msg.Object.(*apiv1.Notebook)
+ m.syncingFiles = inProgress
+ cmds = append(cmds,
+ notebookSyncFilesCmd(m.Ctx, m.Client, m.notebook.DeepCopy(), m.Path),
+ portForwardCmd(m.Ctx, m.Client, client.PodForNotebook(m.notebook), client.ForwardedPorts{Local: 8888, Pod: 8888}),
+ )
+
+ case notebookFileSyncMsg:
+ if msg.complete {
+ m.currentSyncingFile = ""
+ } else {
+ m.currentSyncingFile = msg.file
+ }
+ if msg.error != nil {
+ m.lastSyncFailure = msg.error
+ } else {
+ m.lastSyncFailure = nil
+ }
+
+ case portForwardReadyMsg:
+ cmds = append(cmds, notebookOpenInBrowser(m.notebook.DeepCopy()))
+
+ case localURLMsg:
+ m.localURL = string(msg)
+
+ case tea.WindowSizeMsg:
+ m.Style.Width(msg.Width)
+ innerWidth := m.Style.GetWidth() - m.Style.GetHorizontalPadding()
+ m.upload.Style = lipgloss.NewStyle().Width(innerWidth)
+ m.readiness.Style = lipgloss.NewStyle().Width(innerWidth)
+ m.pods.SetStyle(logStyle.Width(innerWidth))
+
+ case error:
+ log.Printf("Error message: %v", msg)
+ m.finalError = msg
+ m.quitting = true
+ }
+
+ return m, tea.Batch(cmds...)
+}
+
+// View returns a string based on data in the model. That string which will be
+// rendered to the terminal.
+func (m NotebookModel) View() (v string) {
+ defer func() {
+ v = m.Style.Render(v)
+ }()
+
+ if m.finalError != nil {
+ v += errorStyle.Width(m.Style.GetWidth()-m.Style.GetHorizontalMargins()-10).Render("Error: "+m.finalError.Error()) + "\n"
+ v += helpStyle("Press \"s\" to suspend, \"d\" to delete")
+ // v += helpStyle("Press \"l\" to leave be, \"s\" to suspend, \"d\" to delete")
+ return v
+ }
+
+ if m.goodbye != "" {
+ v += m.goodbye + "\n"
+ return v
+ }
+
+ if m.quitting {
+ v += "Quitting...\n"
+ v += helpStyle("Press \"s\" to suspend, \"d\" to delete, \"ESC\" to cancel")
+ // v += helpStyle("Press \"l\" to leave be, \"s\" to suspend, \"d\" to delete, \"ESC\" to cancel")
+ return v
+ }
+
+ v += m.manifests.View()
+ v += m.upload.View()
+ v += m.readiness.View()
+ v += m.pods.View()
+
+ if m.syncingFiles == inProgress {
+ v += "\n"
+ if m.currentSyncingFile != "" {
+ v += fmt.Sprintf("Syncing from notebook: %v\n", m.currentSyncingFile)
+ } else {
+ v += "Watching for files to sync...\n"
+ }
+ if m.lastSyncFailure != nil {
+ v += errorStyle.Render("Sync failed: "+m.lastSyncFailure.Error()) + "\n\n"
+ }
+ }
+
+ if m.portForwarding == inProgress {
+ v += "Port-forwarding...\n"
+ }
+
+ if m.localURL != "" && m.portForwarding == inProgress {
+ v += "\n"
+ v += fmt.Sprintf("Notebook URL: %v\n", m.localURL)
+ }
+
+ v += helpStyle("Press \"q\" to quit")
+
+ return v
+}
+
+type notebookFileSyncMsg struct {
+ file string
+ complete bool
+ error error
+}
+
+func notebookSyncFilesCmd(ctx context.Context, c client.Interface, nb *apiv1.Notebook, dir string) tea.Cmd {
+ return func() tea.Msg {
+ if err := c.SyncFilesFromNotebook(ctx, nb, dir, LogFile, func(file string, complete bool, syncErr error) {
+ P.Send(notebookFileSyncMsg{
+ file: file,
+ complete: complete,
+ error: syncErr,
+ })
+ }); err != nil {
+ if !errors.Is(err, context.Canceled) {
+ return err
+ }
+ }
+ return nil
+ }
+}
+
+func notebookOpenInBrowser(nb *apiv1.Notebook) tea.Cmd {
+ return func() tea.Msg {
+ // TODO(nstogner): Grab token from Notebook status.
+ url := "http://localhost:8888?token=default"
+ log.Printf("Opening browser to %s\n", url)
+ browser.OpenURL(url)
+ return localURLMsg(url)
+ }
+}
diff --git a/internal/tui/pods.go b/internal/tui/pods.go
new file mode 100644
index 00000000..e70eaf21
--- /dev/null
+++ b/internal/tui/pods.go
@@ -0,0 +1,246 @@
+package tui
+
+import (
+ "bufio"
+ "context"
+ "fmt"
+ "log"
+ "sort"
+ "strings"
+
+ "github.com/charmbracelet/bubbles/viewport"
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/charmbracelet/lipgloss"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+ "k8s.io/apimachinery/pkg/watch"
+ "k8s.io/client-go/kubernetes"
+
+ "github.com/substratusai/substratus/internal/client"
+)
+
+type podsModel struct {
+ // Cancellation
+ Ctx context.Context
+
+ // Clients
+ Client client.Interface
+ Resource *client.Resource
+ K8s *kubernetes.Clientset
+
+ Object client.Object
+
+ watchingPods status
+
+ // Watch Pods
+ // map[role][podName]
+ pods map[string]map[string]podInfo
+
+ // End times
+ finalError error
+
+ Style lipgloss.Style
+}
+
+type podInfo struct {
+ lastEvent watch.EventType
+ pod *corev1.Pod
+
+ logs string
+ logsStarted bool
+ logsViewport viewport.Model
+}
+
+// New initializes all internal fields.
+func (m *podsModel) New() podsModel {
+ m.pods = map[string]map[string]podInfo{}
+ return *m
+}
+
+func (m podsModel) Active() bool {
+ return m.watchingPods == inProgress
+}
+
+func (m podsModel) Init() tea.Cmd {
+ return tea.Sequence(
+ func() tea.Msg { return podsInitMsg{} },
+ watchPods(m.Ctx, m.Client, m.Object.DeepCopyObject().(client.Object)),
+ )
+}
+
+type podsInitMsg struct{}
+
+func (m podsModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+ switch msg := msg.(type) {
+ case podsInitMsg:
+ log.Println("Initializing pods")
+ m.watchingPods = inProgress
+
+ case podWatchMsg:
+ role := msg.Pod.Labels["role"]
+
+ if _, ok := m.pods[role][msg.Pod.Name]; !ok {
+ m.pods[role] = map[string]podInfo{}
+ }
+ pi := m.pods[role][msg.Pod.Name]
+ pi.lastEvent = msg.Type
+ pi.pod = msg.Pod.DeepCopy()
+
+ containerName := pi.pod.Annotations["kubectl.kubernetes.io/default-container"]
+
+ var cmd tea.Cmd
+ if !pi.logsStarted {
+ for _, status := range pi.pod.Status.ContainerStatuses {
+ if status.Name == containerName && status.Ready {
+ log.Printf("Getting logs for Pod container: %v", status.Name)
+ cmd = getLogs(m.Ctx, m.K8s, pi.pod, containerName)
+ pi.logsStarted = true
+ pi.logsViewport = viewport.New(m.Style.GetWidth()-10, 7)
+ pi.logsViewport.Style = m.Style
+ break
+ } else {
+ log.Printf("Skipping logs for container: %v (Ready = %v)", status.Name, status.Ready)
+ }
+ }
+ }
+
+ m.pods[msg.Pod.Labels["role"]][msg.Pod.Name] = pi
+ return m, cmd
+
+ case podLogsMsg:
+ pi := m.pods[msg.role][msg.name]
+ // Fix the rendering of line-rewrites by always appending lines.
+ logs := msg.logs
+ logs = strings.ReplaceAll(logs, "\r", "\n")
+ logs = strings.TrimRight(logs, "\n")
+ logs = logs + "\n"
+ pi.logs += logs
+ pi.logsViewport.SetContent(lipgloss.NewStyle().Width(m.Style.GetWidth() - m.Style.GetHorizontalPadding()).Render(pi.logs) /*wordwrap.String(pi.logs, m.width-14)*/)
+ pi.logsViewport.GotoBottom()
+ m.pods[msg.role][msg.name] = pi
+ return m, nil
+
+ case error:
+ m.finalError = msg
+ return m, nil
+ }
+
+ return m, nil
+}
+
+// View returns a string based on data in the model. That string which will be
+// rendered to the terminal.
+func (m podsModel) View() (v string) {
+ if m.watchingPods == inProgress {
+ v += "Pods:\n"
+
+ roles := []string{"build", "run"}
+
+ for _, role := range roles {
+ var pods []podInfo
+ for _, p := range m.pods[role] {
+ pods = append(pods, p)
+ }
+ sort.Slice(pods, func(i, j int) bool {
+ return pods[i].pod.CreationTimestamp.Before(&pods[j].pod.CreationTimestamp)
+ })
+ for _, p := range pods {
+ if p.lastEvent == watch.Deleted {
+ continue
+ }
+ v += "> " + strings.Title(p.pod.Labels["role"]) + " (" + string(p.pod.Status.Phase) + ")\n"
+ if p.pod.Status.Phase != corev1.PodSucceeded {
+ v += "\n" + p.logsViewport.View() + "\n"
+ }
+ }
+ }
+ }
+
+ return v
+}
+
+func (m *podsModel) SetStyle(s lipgloss.Style) {
+ m.Style = s
+ for role := range m.pods {
+ for name := range m.pods[role] {
+ pi := m.pods[role][name]
+ if pi.logsViewport.Width > 0 {
+ pi.logsViewport.Width = s.GetWidth()
+ pi.logsViewport.SetContent(lipgloss.NewStyle().Width(s.GetWidth() - s.GetHorizontalMargins()).Render(pi.logs))
+ pi.logsViewport.Style = s
+ m.pods[role][name] = pi
+ }
+ }
+ }
+}
+
+type podWatchMsg struct {
+ Type watch.EventType
+ Pod *corev1.Pod
+}
+
+func watchPods(ctx context.Context, c client.Interface, obj client.Object) tea.Cmd {
+ return func() tea.Msg {
+ log.Println("Starting Pod watch")
+
+ pods, err := c.Resource(&corev1.Pod{TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}})
+ if err != nil {
+ return fmt.Errorf("pods client: %w", err)
+ }
+
+ w, err := pods.Watch(ctx, obj.GetNamespace(), nil, &metav1.ListOptions{
+ LabelSelector: labels.SelectorFromSet(map[string]string{
+ strings.ToLower(obj.GetObjectKind().GroupVersionKind().Kind): obj.GetName(),
+ //"role": role,
+ }).String(),
+ })
+ if err != nil {
+ return fmt.Errorf("watch: %w", err)
+ }
+ go func() {
+ for event := range w.ResultChan() {
+ switch event.Type {
+ case watch.Added, watch.Modified, watch.Deleted:
+ pod := event.Object.(*corev1.Pod)
+ log.Printf("Pod event: %s: %s", pod.Name, event.Type)
+ P.Send(podWatchMsg{Type: event.Type, Pod: pod})
+ }
+ }
+ }()
+
+ return nil
+ }
+}
+
+type podLogsMsg struct {
+ role string
+ name string
+ logs string
+}
+
+func getLogs(ctx context.Context, k8s *kubernetes.Clientset, pod *corev1.Pod, container string) tea.Cmd {
+ return func() tea.Msg {
+ log.Printf("Starting to get logs for pod: %v", pod.Name)
+ req := k8s.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{
+ Container: container,
+ Follow: true,
+ Timestamps: false,
+ })
+ logs, err := req.Stream(ctx)
+ if err != nil {
+ return err
+ }
+
+ scanner := bufio.NewScanner(logs)
+ for scanner.Scan() {
+ logs := scanner.Text()
+ log.Printf("Pod logs for: %v: %q", pod.Name, logs)
+ P.Send(podLogsMsg{role: pod.Labels["role"], name: pod.Name, logs: logs})
+ }
+ if err := scanner.Err(); err != nil {
+ return err
+ }
+ return nil
+ }
+}
diff --git a/internal/tui/portforward.go b/internal/tui/portforward.go
new file mode 100644
index 00000000..f8eb503f
--- /dev/null
+++ b/internal/tui/portforward.go
@@ -0,0 +1,63 @@
+package tui
+
+import (
+ "context"
+ "log"
+ "math"
+ "time"
+
+ tea "github.com/charmbracelet/bubbletea"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/util/runtime"
+
+ "github.com/substratusai/substratus/internal/client"
+)
+
+type portForwardReadyMsg struct{}
+
+func portForwardCmd(ctx context.Context, c client.Interface, podRef types.NamespacedName, ports client.ForwardedPorts) tea.Cmd {
+ return func() tea.Msg {
+ const maxRetries = 3
+ for i := 0; i < maxRetries; i++ {
+ portFwdCtx, cancelPortFwd := context.WithCancel(ctx)
+ defer cancelPortFwd() // Avoid a context leak
+ runtime.ErrorHandlers = []func(err error){
+ func(err error) {
+ // Cancel a broken port forward to attempt to restart the port-forward.
+ log.Printf("Port-forward error: %v", err)
+ cancelPortFwd()
+ },
+ }
+
+ // portForward will close the ready channel when it returns.
+ // so we only use the outer ready channel once. On restart of the portForward,
+ // we use a new channel.
+ ready := make(chan struct{})
+ go func() {
+ log.Println("Waiting for port-forward to be ready")
+ <-ready
+ log.Println("Port-forward ready")
+ P.Send(portForwardReadyMsg{})
+ }()
+
+ if err := c.PortForward(portFwdCtx, LogFile, podRef, ports, ready); err != nil {
+ log.Printf("Port-forward returned an error: %v", err)
+ }
+
+ // Check if the command's context is cancelled, if so,
+ // avoid restarting the port forward.
+ if err := ctx.Err(); err != nil {
+ log.Printf("Context done, not attempting to restart port-forward: %v", err.Error())
+ return nil
+ }
+
+ cancelPortFwd() // Avoid a build up of contexts before returning.
+ backoff := time.Duration(math.Pow(2, float64(i))) * time.Second
+ log.Printf("Restarting port forward (index = %v), after backoff: %s", i, backoff)
+ time.Sleep(backoff)
+ }
+ log.Println("Done trying to port-forward")
+
+ return nil
+ }
+}
diff --git a/internal/tui/readiness.go b/internal/tui/readiness.go
new file mode 100644
index 00000000..70db4b6b
--- /dev/null
+++ b/internal/tui/readiness.go
@@ -0,0 +1,100 @@
+package tui
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/charmbracelet/lipgloss"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "github.com/substratusai/substratus/internal/client"
+)
+
+type readinessModel struct {
+ // Cancellation
+ Ctx context.Context
+
+ // Clients
+ Client client.Interface
+ Resource *client.Resource
+
+ Object client.Object
+
+ // Readiness
+ waiting status
+
+ Style lipgloss.Style
+}
+
+// New initializes all internal fields.
+func (m *readinessModel) New() readinessModel {
+ m.Style = lipgloss.NewStyle()
+ return *m
+}
+
+func (m readinessModel) Active() bool {
+ return m.waiting == inProgress
+}
+
+func (m readinessModel) Init() tea.Cmd {
+ return tea.Sequence(
+ func() tea.Msg { return readinessInitMsg{} },
+ waitReadyCmd(m.Ctx, m.Resource, m.Object.DeepCopyObject().(client.Object)),
+ )
+}
+
+type readinessInitMsg struct{}
+
+func (m readinessModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+ switch msg := msg.(type) {
+ case readinessInitMsg:
+ log.Println("Initializing readiness")
+ m.waiting = inProgress
+
+ case objectUpdateMsg:
+ m.Object = msg.Object
+
+ case objectReadyMsg:
+ m.waiting = completed
+ m.Object = msg.Object
+ }
+
+ return m, nil
+}
+
+// View returns a string based on data in the model. That string which will be
+// rendered to the terminal.
+func (m readinessModel) View() (v string) {
+ defer func() {
+ if v != "" {
+ v = m.Style.Render(v)
+ }
+ }()
+
+ if m.waiting == inProgress {
+ kind := m.Object.GetObjectKind().GroupVersionKind().Kind
+ v += fmt.Sprintf("%v:\n", kind)
+
+ if w, ok := m.Object.(interface {
+ GetConditions() *[]metav1.Condition
+ }); ok {
+ for _, c := range *w.GetConditions() {
+ var prefix, suffix string
+ if c.Status == metav1.ConditionTrue {
+ prefix = checkMark.String() + " "
+ } else {
+ prefix = xMark.String() + " "
+ suffix = " (" + c.Reason + ")"
+ }
+ v += lipgloss.NewStyle().Width(m.Style.GetWidth() - m.Style.GetHorizontalPadding()).
+ Render(prefix + c.Type + suffix)
+ v += "\n"
+ }
+ }
+
+ }
+
+ return v
+}
diff --git a/internal/tui/run.go b/internal/tui/run.go
new file mode 100644
index 00000000..fb7092a0
--- /dev/null
+++ b/internal/tui/run.go
@@ -0,0 +1,176 @@
+package tui
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log"
+
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/charmbracelet/lipgloss"
+ "k8s.io/client-go/kubernetes"
+
+ "github.com/substratusai/substratus/internal/client"
+)
+
+type RunModel struct {
+ // Cancellation
+ Ctx context.Context
+
+ // Clients
+ K8s *kubernetes.Clientset
+ Client client.Interface
+
+ // Config
+ Path string
+ Filename string
+ Namespace Namespace
+
+ // Focal object
+ object client.Object
+ resource *client.Resource
+
+ // Processes
+ manifests manifestsModel
+ upload uploadModel
+ readiness readinessModel
+ pods podsModel
+
+ // End times
+ finalError error
+
+ Style lipgloss.Style
+}
+
+func (m *RunModel) New() RunModel {
+ m.manifests = (&manifestsModel{
+ Path: m.Path,
+ Filename: m.Filename,
+ Kinds: []string{"Model", "Dataset"},
+ }).New()
+ m.upload = (&uploadModel{
+ Ctx: m.Ctx,
+ Client: m.Client,
+ Path: m.Path,
+ Mode: uploadModeCreate,
+ }).New()
+ m.readiness = (&readinessModel{
+ Ctx: m.Ctx,
+ Client: m.Client,
+ }).New()
+ m.pods = (&podsModel{
+ Ctx: m.Ctx,
+ Client: m.Client,
+ K8s: m.K8s,
+ }).New()
+ m.Style = appStyle
+ return *m
+}
+
+func (m RunModel) Init() tea.Cmd {
+ return m.manifests.Init()
+}
+
+func (m RunModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+ var cmds []tea.Cmd
+
+ log.Printf("MSG: %T", msg)
+ {
+ mdl, cmd := m.manifests.Update(msg)
+ m.manifests = mdl.(manifestsModel)
+ cmds = append(cmds, cmd)
+ }
+
+ {
+ mdl, cmd := m.upload.Update(msg)
+ m.upload = mdl.(uploadModel)
+ cmds = append(cmds, cmd)
+ }
+
+ {
+ mdl, cmd := m.readiness.Update(msg)
+ m.readiness = mdl.(readinessModel)
+ cmds = append(cmds, cmd)
+ }
+
+ {
+ mdl, cmd := m.pods.Update(msg)
+ m.pods = mdl.(podsModel)
+ cmds = append(cmds, cmd)
+ }
+
+ switch msg := msg.(type) {
+ case manifestSelectedMsg:
+ m.object = msg.obj
+ m.Namespace.Set(m.object)
+
+ res, err := m.Client.Resource(m.object)
+ if err != nil {
+ b, _ := json.Marshal(m.object)
+ log.Println("............", string(b))
+
+ m.finalError = fmt.Errorf("resource client: %w", err)
+ break
+ }
+ m.resource = res
+
+ m.upload.Object = m.object
+ m.upload.Resource = m.resource
+ cmds = append(cmds, m.upload.Init())
+
+ case tea.KeyMsg:
+ log.Println("Received key msg:", msg.String())
+ if msg.String() == "q" {
+ cmds = append(cmds, tea.Quit)
+ }
+
+ case tarballUploadedMsg:
+ m.object = msg.Object
+
+ m.readiness.Object = m.object
+ m.readiness.Resource = m.resource
+ m.pods.Object = m.object
+ m.pods.Resource = m.resource
+ cmds = append(cmds,
+ m.readiness.Init(),
+ m.pods.Init(),
+ )
+
+ case tea.WindowSizeMsg:
+ m.Style.Width(msg.Width)
+ innerWidth := m.Style.GetWidth() - m.Style.GetHorizontalPadding()
+ // NOTE: Use background coloring for style debugging.
+ m.upload.Style = lipgloss.NewStyle().Width(innerWidth) //.Background(lipgloss.Color("12"))
+ m.readiness.Style = lipgloss.NewStyle().Width(innerWidth) //.Background(lipgloss.Color("202"))
+ m.pods.SetStyle(logStyle.Copy().Width(innerWidth)) //.Background(lipgloss.Color("86")))
+
+ case error:
+ log.Printf("Error message: %v", msg)
+ m.finalError = msg
+ }
+
+ return m, tea.Batch(cmds...)
+}
+
+// View returns a string based on data in the model. That string which will be
+// rendered to the terminal.
+func (m RunModel) View() (v string) {
+ defer func() {
+ v = m.Style.Render(v)
+ }()
+
+ if m.finalError != nil {
+ v += errorStyle.Width(m.Style.GetWidth()-m.Style.GetHorizontalPadding()).Render("Error: "+m.finalError.Error()) + "\n"
+ v += helpStyle("Press \"q\" to quit")
+ return v
+ }
+
+ v += m.manifests.View()
+ v += m.upload.View()
+ v += m.readiness.View()
+ v += m.pods.View()
+
+ v += helpStyle("Press \"q\" to quit")
+
+ return v
+}
diff --git a/internal/tui/serve.go b/internal/tui/serve.go
new file mode 100644
index 00000000..dfe390e5
--- /dev/null
+++ b/internal/tui/serve.go
@@ -0,0 +1,281 @@
+package tui
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/charmbracelet/lipgloss"
+ "github.com/pkg/browser"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/client-go/kubernetes"
+
+ apiv1 "github.com/substratusai/substratus/api/v1"
+ "github.com/substratusai/substratus/internal/client"
+)
+
+type ServeModel struct {
+ // Cancellation
+ Ctx context.Context
+
+ // Config
+ Namespace Namespace
+ Filename string
+ NoOpenBrowser bool
+
+ // Clients
+ Client client.Interface
+ K8s *kubernetes.Clientset
+
+ // Current Server
+ server *apiv1.Server
+ resource *client.Resource
+ readyPod *corev1.Pod
+
+ applying status
+
+ manifests manifestsModel
+ readiness readinessModel
+ pods podsModel
+
+ // Ready to open browser
+ portForwarding status
+ localURL string
+
+ Style lipgloss.Style
+
+ // End times
+ quitting bool
+ goodbye string
+ finalError error
+}
+
+func (m *ServeModel) New() ServeModel {
+ m.manifests = (&manifestsModel{
+ Kinds: []string{"Server"},
+ }).New()
+ m.readiness = (&readinessModel{
+ Ctx: m.Ctx,
+ Client: m.Client,
+ }).New()
+ m.pods = (&podsModel{
+ Ctx: m.Ctx,
+ Client: m.Client,
+ K8s: m.K8s,
+ }).New()
+
+ m.Style = appStyle
+
+ return *m
+}
+
+func (m ServeModel) Init() tea.Cmd {
+ return m.manifests.Init()
+}
+
+func (m ServeModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+ var cmds []tea.Cmd
+
+ log.Printf("MSG: %T", msg)
+
+ {
+ mdl, cmd := m.manifests.Update(msg)
+ m.manifests = mdl.(manifestsModel)
+ cmds = append(cmds, cmd)
+ }
+
+ {
+ mdl, cmd := m.readiness.Update(msg)
+ m.readiness = mdl.(readinessModel)
+ cmds = append(cmds, cmd)
+ }
+
+ {
+ mdl, cmd := m.pods.Update(msg)
+ m.pods = mdl.(podsModel)
+ cmds = append(cmds, cmd)
+ }
+
+ apply := func() {
+ res, err := m.Client.Resource(m.server)
+ if err != nil {
+ m.finalError = fmt.Errorf("resource client: %w", err)
+ return
+ }
+ m.resource = res
+
+ m.applying = inProgress
+ cmds = append(cmds, applyCmd(m.Ctx, m.resource, m.server.DeepCopy()))
+ }
+
+ switch msg := msg.(type) {
+ case manifestSelectedMsg:
+ m.Namespace.Set(msg.obj)
+ m.server = msg.obj.(*apiv1.Server)
+ apply()
+
+ case appliedMsg:
+ m.applying = completed
+ m.server = msg.Object.(*apiv1.Server)
+
+ m.readiness.Object = m.server
+ m.readiness.Resource = m.resource
+
+ m.pods.Object = m.server
+ m.pods.Resource = m.resource
+
+ cmds = append(cmds,
+ m.readiness.Init(),
+ m.pods.Init(),
+ )
+
+ case tea.KeyMsg:
+ log.Println("Received key msg:", msg.String())
+ if m.quitting {
+ switch msg.String() {
+ case "esc":
+ if m.finalError == nil {
+ m.quitting = false
+ }
+
+ case "l":
+ cmds = append(cmds, tea.Quit)
+ // case "s":
+ // cmds = append(cmds, suspendCmd(context.Background(), m.resource, m.server))
+ case "d":
+ cmds = append(cmds, deleteCmd(context.Background(), m.resource, m.server))
+ }
+ } else {
+ if msg.String() == "q" {
+ m.quitting = true
+ }
+ }
+
+ // case suspendedMsg:
+ // if msg.error != nil {
+ // m.finalError = msg.error
+ // } else {
+ // m.goodbye = "Server suspended."
+ // }
+ // cmds = append(cmds, tea.Quit)
+
+ case deletedMsg:
+ if msg.error != nil {
+ m.finalError = msg.error
+ } else {
+ m.goodbye = "Server deleted."
+ }
+ cmds = append(cmds, tea.Quit)
+
+ case objectReadyMsg:
+ m.server = msg.Object.(*apiv1.Server)
+ // TODO: What to do?
+ // cmds = append(cmds) // TODO: Port-forward to Pod.
+ // portForwardCmd(m.Ctx, m.Client, client.PodForNotebook(m.Server)),
+ //
+ case podWatchMsg:
+ if m.readyPod != nil {
+ break
+ }
+ if msg.Pod.Labels == nil || msg.Pod.Labels["role"] != "run" {
+ break
+ }
+
+ var ready bool
+ for _, c := range msg.Pod.Status.Conditions {
+ if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue {
+ ready = true
+ break
+ }
+ }
+
+ if ready {
+ m.readyPod = msg.Pod.DeepCopy()
+ cmds = append(cmds,
+ portForwardCmd(m.Ctx, m.Client,
+ types.NamespacedName{Namespace: m.readyPod.Namespace, Name: m.readyPod.Name},
+ client.ForwardedPorts{Local: 8000, Pod: 8080},
+ ),
+ )
+ }
+
+ case portForwardReadyMsg:
+ cmds = append(cmds, serverOpenInBrowser(m.server.DeepCopy()))
+
+ case localURLMsg:
+ m.localURL = string(msg)
+
+ case tea.WindowSizeMsg:
+ m.Style.Width(msg.Width)
+ innerWidth := m.Style.GetWidth() - m.Style.GetHorizontalPadding()
+ m.readiness.Style = lipgloss.NewStyle().Width(innerWidth)
+ m.pods.SetStyle(logStyle.Width(innerWidth))
+
+ case error:
+ log.Printf("Error message: %v", msg)
+ m.finalError = msg
+ m.quitting = true
+ }
+
+ return m, tea.Batch(cmds...)
+}
+
+// View returns a string based on data in the model. That string which will be
+// rendered to the terminal.
+func (m ServeModel) View() (v string) {
+ defer func() {
+ v = m.Style.Render(v)
+ }()
+
+ if m.finalError != nil {
+ v += errorStyle.Width(m.Style.GetWidth()-m.Style.GetHorizontalMargins()-10).Render("Error: "+m.finalError.Error()) + "\n"
+ // v += helpStyle("Press \"s\" to suspend, \"d\" to delete")
+ v += helpStyle("Press \"l\" to leave be, \"d\" to delete")
+ return
+ }
+
+ if m.goodbye != "" {
+ v += m.goodbye + "\n"
+ return
+ }
+
+ if m.quitting {
+ v += "Quitting...\n"
+ v += helpStyle("Press \"l\" to leave be, \"d\" to delete, \"ESC\" to cancel")
+ // v += helpStyle("Press \"s\" to suspend, \"d\" to delete, \"ESC\" to cancel")
+ // v += helpStyle("Press \"l\" to leave be, \"s\" to suspend, \"d\" to delete, \"ESC\" to cancel")
+ return
+ }
+
+ if m.applying == inProgress {
+ v += "Applying...\n\n"
+ }
+
+ v += m.manifests.View()
+ v += m.readiness.View()
+ v += m.pods.View()
+
+ if m.portForwarding == inProgress {
+ v += "Port-forwarding...\n"
+ }
+
+ if m.localURL != "" && m.portForwarding == inProgress {
+ v += "\n"
+ v += fmt.Sprintf("Server URL: %v\n", m.localURL)
+ }
+
+ v += helpStyle("Press \"q\" to quit")
+
+ return v
+}
+
+func serverOpenInBrowser(s *apiv1.Server) tea.Cmd {
+ return func() tea.Msg {
+ url := "http://localhost:8000"
+ log.Printf("Opening browser to %s\n", url)
+ browser.OpenURL(url)
+ return localURLMsg(url)
+ }
+}
diff --git a/internal/tui/styles.go b/internal/tui/styles.go
new file mode 100644
index 00000000..c169d921
--- /dev/null
+++ b/internal/tui/styles.go
@@ -0,0 +1,33 @@
+package tui
+
+import "github.com/charmbracelet/lipgloss"
+
+var (
+ appStyle = lipgloss.NewStyle().
+ PaddingTop(1).
+ PaddingRight(2).
+ PaddingBottom(1).
+ PaddingLeft(2)
+
+ podStyle = lipgloss.NewStyle().PaddingLeft(2).Render
+
+ // https://coolors.co/palette/264653-2a9d8f-e9c46a-f4a261-e76f51
+ //
+ logStyle = lipgloss.NewStyle().PaddingLeft(1).Border(lipgloss.NormalBorder(), false, false, false, true) /*lipgloss.Border{
+ TopLeft: "| ",
+ BottomLeft: "| ",
+ Left: "| ",
+ })*/
+
+ helpStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#626262")).
+ MarginTop(1).
+ MarginBottom(1).
+ Render
+
+ errorStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#e76f51"))
+
+ activeSpinnerStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#E9C46A"))
+ checkMark = lipgloss.NewStyle().Foreground(lipgloss.Color("#2a9d8f")).SetString("✓")
+ // TODO: Better X mark?
+ xMark = lipgloss.NewStyle().Foreground(lipgloss.Color("#e76f51")).SetString("x")
+)
diff --git a/internal/tui/upload.go b/internal/tui/upload.go
new file mode 100644
index 00000000..61f8b4f3
--- /dev/null
+++ b/internal/tui/upload.go
@@ -0,0 +1,169 @@
+package tui
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+
+ "github.com/charmbracelet/bubbles/progress"
+ tea "github.com/charmbracelet/bubbletea"
+ "github.com/charmbracelet/lipgloss"
+
+ "github.com/substratusai/substratus/internal/client"
+)
+
+type uploadModel struct {
+ // Cancellation
+ Ctx context.Context
+
+ // Config
+ Path string
+
+ // Clients
+ Client client.Interface
+ Resource *client.Resource
+
+ Mode uploadMode
+ applying status
+ creating status
+
+ // Original Object (could be a Dataset, Model, or Server)
+ Object client.Object
+
+ // Tarring
+ tarring status
+ tarredFileCount int
+ tarball *client.Tarball
+
+ // Uploading
+ uploading status
+ uploadProgress progress.Model
+
+ Style lipgloss.Style
+}
+
+type uploadMode int
+
+const (
+ uploadModeCreate = 0
+ uploadModeApply = 1
+)
+
+func (m uploadModel) kind() string {
+ return m.Object.GetObjectKind().GroupVersionKind().Kind
+}
+
+func (m uploadModel) cleanup() {
+ log.Println("Cleaning up")
+ os.Remove(m.tarball.TempDir)
+}
+
+// New initializes all internal fields.
+func (m *uploadModel) New() uploadModel {
+ m.Style = lipgloss.NewStyle()
+ m.uploadProgress = progress.New(progress.WithDefaultGradient())
+ return *m
+}
+
+func (m uploadModel) Active() bool {
+ return true
+ //if m.Mode == uploadModeApply {
+ // return m.applying != completed
+ //} else if m.Mode == uploadModeCreate {
+ // return m.creating != completed
+ //} else {
+ // panic("Unrecognized mode")
+ //}
+}
+
+func (m uploadModel) Init() tea.Cmd {
+ return tea.Sequence(
+ func() tea.Msg { return uploadInitMsg{} },
+ prepareTarballCmd(m.Ctx, m.Path),
+ )
+}
+
+type uploadInitMsg struct{}
+
+func (m uploadModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+ switch msg := msg.(type) {
+ case uploadInitMsg:
+ m.tarring = inProgress
+
+ case fileTarredMsg:
+ m.tarredFileCount++
+ return m, nil
+
+ case tarballCompleteMsg:
+ m.tarring = completed
+ m.tarball = msg
+ if m.Mode == uploadModeApply {
+ m.applying = inProgress
+ return m, applyWithUploadCmd(m.Ctx, m.Resource, m.Object.DeepCopyObject().(client.Object), m.tarball)
+ } else if m.Mode == uploadModeCreate {
+ m.creating = inProgress
+ return m, createWithUploadCmd(m.Ctx, m.Resource, m.Object.DeepCopyObject().(client.Object), m.tarball)
+ } else {
+ panic("unkown upload mode")
+ }
+
+ case appliedWithUploadMsg:
+ m.Object = msg.Object
+ m.applying = completed
+ m.uploading = inProgress
+ return m, uploadTarballCmd(m.Ctx, m.Resource, m.Object.DeepCopyObject().(client.Object), m.tarball)
+
+ case createdWithUploadMsg:
+ m.Object = msg.Object
+ m.creating = completed
+ m.uploading = inProgress
+ return m, uploadTarballCmd(m.Ctx, m.Resource, m.Object.DeepCopyObject().(client.Object), m.tarball)
+
+ case uploadTarballProgressMsg:
+ return m, m.uploadProgress.SetPercent(float64(msg))
+
+ case tarballUploadedMsg:
+ m.uploading = completed
+ m.Object = msg.Object
+
+ // FrameMsg is sent when the progress bar wants to animate itself
+ case progress.FrameMsg:
+ progressModel, cmd := m.uploadProgress.Update(msg)
+ m.uploadProgress = progressModel.(progress.Model)
+ return m, cmd
+ }
+
+ return m, nil
+}
+
+// View returns a string based on data in the model. That string which will be
+// rendered to the terminal.
+func (m uploadModel) View() (v string) {
+ defer func() {
+ if v != "" {
+ v = m.Style.Render(v)
+ }
+ }()
+ m.uploadProgress.Width = m.Style.GetWidth()
+
+ if m.tarring == inProgress {
+ v += "Tarring...\n"
+ v += fmt.Sprintf("File count: %v\n", m.tarredFileCount)
+ }
+
+ if m.applying == inProgress {
+ v += "Applying...\n"
+ }
+
+ if m.creating == inProgress {
+ v += "Creating...\n"
+ }
+
+ if m.uploading == inProgress {
+ v += "Uploading...\n\n"
+ v += m.uploadProgress.View() + "\n\n"
+ }
+
+ return v
+}
diff --git a/kubectl/cmd/applybuild/main.go b/kubectl/cmd/applybuild/main.go
deleted file mode 100644
index 93c2dd38..00000000
--- a/kubectl/cmd/applybuild/main.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package main
-
-import (
- "github.com/substratusai/substratus/kubectl/internal/commands"
- "k8s.io/klog/v2"
-)
-
-var Version = "development"
-
-func main() {
- commands.Version = Version
- if err := commands.ApplyBuild().Execute(); err != nil {
- klog.Fatal(err)
- }
-}
diff --git a/kubectl/cmd/notebook/main.go b/kubectl/cmd/notebook/main.go
deleted file mode 100644
index 293be1f6..00000000
--- a/kubectl/cmd/notebook/main.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package main
-
-import (
- "github.com/substratusai/substratus/kubectl/internal/commands"
- "k8s.io/klog/v2"
-)
-
-var Version = "development"
-
-func main() {
- commands.Version = Version
- if err := commands.Notebook().Execute(); err != nil {
- klog.Fatal(err)
- }
-}
diff --git a/kubectl/internal/client/notebook.go b/kubectl/internal/client/notebook.go
deleted file mode 100644
index eb5525b0..00000000
--- a/kubectl/internal/client/notebook.go
+++ /dev/null
@@ -1,84 +0,0 @@
-package client
-
-import (
- "fmt"
-
- apiv1 "github.com/substratusai/substratus/api/v1"
-
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-func NotebookForObject(obj Object) (*apiv1.Notebook, error) {
- var nb *apiv1.Notebook
-
- switch obj := obj.DeepCopyObject().(type) {
- case *apiv1.Notebook:
- nb = obj
-
- case *apiv1.Model:
- return nil, fmt.Errorf("notebooks for models are not yet supported")
- /*
- nb = &apiv1.Notebook{
- ObjectMeta: metav1.ObjectMeta{
- Name: obj.Name,
- Namespace: obj.Namespace,
- },
- Spec: apiv1.NotebookSpec{
- // TODO: How to map base model / saved model to notebook mounts?
- Image: obj.Spec.Image,
- Params: obj.Spec.Params,
- },
- }
- */
-
- case *apiv1.Server:
- return nil, fmt.Errorf("notebooks for servers are not yet supported")
- /*
- nb = &apiv1.Notebook{
- ObjectMeta: metav1.ObjectMeta{
- Name: obj.Name,
- Namespace: obj.Namespace,
- },
- Spec: apiv1.NotebookSpec{
- Image: obj.Spec.Image,
- Model: &obj.Spec.Model,
- },
- }
- */
-
- case *apiv1.Dataset:
- return nil, fmt.Errorf("notebooks for datasets are not yet supported")
- // NOTE: For this to work for Dataset development purposes, the Notebook
- // controllers needs to mount a directory to receive the dataset.
- // (/content/data).
- /*
- nb = &apiv1.Notebook{
- ObjectMeta: metav1.ObjectMeta{
- Name: obj.Name,
- Namespace: obj.Namespace,
- },
- Spec: apiv1.NotebookSpec{
- Build: obj.Spec.Build,
- //Dataset: &apiv1.ObjectRef{
- // Name: obj.Name,
- //},
- Params: obj.Spec.Params,
- },
- }
- */
-
- default:
- return nil, fmt.Errorf("unknown object type: %T", obj)
- }
-
- // "This field is managed by the API server and should not be changed by the user."
- // https://kubernetes.io/docs/reference/using-api/server-side-apply/#field-management
- nb.ObjectMeta.ManagedFields = nil
-
- nb.TypeMeta = metav1.TypeMeta{
- APIVersion: "substratus.ai/v1",
- Kind: "Notebook",
- }
-
- return nb, nil
-}
diff --git a/kubectl/internal/commands/applybuild.go b/kubectl/internal/commands/applybuild.go
deleted file mode 100644
index 78d9492d..00000000
--- a/kubectl/internal/commands/applybuild.go
+++ /dev/null
@@ -1,129 +0,0 @@
-package commands
-
-import (
- "flag"
- "fmt"
- "os"
-
- "github.com/spf13/cobra"
- "k8s.io/client-go/kubernetes"
- "k8s.io/client-go/tools/clientcmd"
- "k8s.io/klog/v2"
-
- "github.com/substratusai/substratus/kubectl/internal/client"
-)
-
-func ApplyBuild() *cobra.Command {
- var flags struct {
- namespace string
- filename string
- build string
- kubeconfig string
- forceConflicts bool
- }
-
- cmd := &cobra.Command{
- Use: "applybuild [flags] BUILD_CONTEXT",
- Args: cobra.ExactArgs(1),
- Short: "Apply a Substratus object, upload and build container in-cluster from a local directory",
- Version: Version,
- RunE: func(cmd *cobra.Command, args []string) error {
- client.Version = Version
-
- ctx := cmd.Context()
-
- if flags.filename == "" {
- return fmt.Errorf("-f (--filename) is required")
- }
- flags.build = args[0]
-
- tarball, err := client.PrepareImageTarball(ctx, flags.build)
- if err != nil {
- return fmt.Errorf("preparing tarball: %w", err)
- }
- defer os.Remove(tarball.TempDir)
-
- kubeconfigNamespace, restConfig, err := buildConfigFromFlags("", flags.kubeconfig)
- if err != nil {
- return fmt.Errorf("rest config: %w", err)
- }
-
- namespace := "default"
- if flags.namespace != "" {
- namespace = flags.namespace
- } else if kubeconfigNamespace != "" {
- namespace = kubeconfigNamespace
- }
-
- clientset, err := kubernetes.NewForConfig(restConfig)
- if err != nil {
- return fmt.Errorf("clientset: %w", err)
- }
-
- manifest, err := os.ReadFile(flags.filename)
- if err != nil {
- return fmt.Errorf("reading file: %w", err)
- }
-
- obj, err := client.Decode(manifest)
- if err != nil {
- return fmt.Errorf("decoding: %w", err)
- }
-
- if obj.GetNamespace() == "" {
- // When there is no .metadata.namespace set in the manifest...
- obj.SetNamespace(namespace)
- } else {
- // TODO: Closer match kubectl behavior here by differentiaing between
- // the short -n and long --namespace flags.
- // See example kubectl error:
- // error: the namespace from the provided object "a" does not match the namespace "b". You must pass '--namespace=a' to perform this operation.
- if flags.namespace != "" && flags.namespace != obj.GetNamespace() {
- // When there is .metadata.namespace set in the manifest and
- // a conflicting -n or --namespace flag...
- return fmt.Errorf("the namespace from the provided object %q does not match the namespace %q from flag", obj.GetNamespace(), flags.namespace)
- }
- }
-
- c := NewClient(clientset, restConfig)
- r, err := c.Resource(obj)
- if err != nil {
- return fmt.Errorf("resource client: %w", err)
- }
-
- if err := client.SetUploadContainerSpec(obj, tarball, NewUUID()); err != nil {
- return fmt.Errorf("setting upload in spec: %w", err)
- }
- if err := client.ClearImage(obj); err != nil {
- return fmt.Errorf("clearing image: %w", err)
- }
-
- if err := r.Apply(obj, flags.forceConflicts); err != nil {
- return fmt.Errorf("applying: %w", err)
- }
-
- if err := r.Upload(ctx, obj, tarball); err != nil {
- return fmt.Errorf("uploading: %w", err)
- }
-
- return nil
- },
- }
-
- defaultKubeconfig := os.Getenv("KUBECONFIG")
- if defaultKubeconfig == "" {
- defaultKubeconfig = clientcmd.RecommendedHomeFile
- }
- cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "")
- cmd.Flags().StringVarP(&flags.filename, "filename", "f", "", "Filename identifying the resource to apply and build.")
- cmd.Flags().BoolVar(&flags.forceConflicts, "force-conflicts", false, "If true, server-side apply will force the changes against conflicts.")
-
- cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook")
-
- // Add standard kubectl logging flags (for example: -v=2).
- goflags := flag.NewFlagSet("", flag.PanicOnError)
- klog.InitFlags(goflags)
- cmd.Flags().AddGoFlagSet(goflags)
-
- return cmd
-}
diff --git a/kubectl/internal/commands/applybuild_test.go b/kubectl/internal/commands/applybuild_test.go
deleted file mode 100644
index e7f575ed..00000000
--- a/kubectl/internal/commands/applybuild_test.go
+++ /dev/null
@@ -1,72 +0,0 @@
-package commands_test
-
-import (
- "net/http"
- "net/http/httptest"
- "sync"
- "testing"
-
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- apiv1 "github.com/substratusai/substratus/api/v1"
- "github.com/substratusai/substratus/kubectl/internal/commands"
- "k8s.io/apimachinery/pkg/types"
-)
-
-func TestApplyBuild(t *testing.T) {
- cmd := commands.ApplyBuild()
- cmd.SetArgs([]string{
- "--filename", "./test-applybuild/notebook.yaml",
- "--kubeconfig", kubectlKubeconfigPath,
- //"-v=9",
- "./test-applybuild",
- })
- var wg sync.WaitGroup
-
- wg.Add(1)
- go func() {
- defer wg.Done()
- if err := cmd.Execute(); err != nil {
- t.Error(err)
- }
- }()
-
- var uploadedPath string
- var uploadedPathMtx sync.Mutex
- mockBucketServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- t.Log("mockBucketServer handler called")
-
- uploadedPathMtx.Lock()
- uploadedPath = r.URL.String()
- uploadedPathMtx.Unlock()
- }))
- defer mockBucketServer.Close()
-
- nb := &apiv1.Notebook{
- ObjectMeta: metav1.ObjectMeta{
- Name: "test-applybuild",
- Namespace: "default",
- },
- }
- require.EventuallyWithT(t, func(t *assert.CollectT) {
- err := k8sClient.Get(ctx, types.NamespacedName{Namespace: nb.Namespace, Name: nb.Name}, nb)
- assert.NoError(t, err, "getting notebook")
- }, timeout, interval, "waiting for the notebook to be created")
-
- nb.Status.BuildUpload = apiv1.UploadStatus{
- SignedURL: mockBucketServer.URL + "/some-signed-url",
- RequestID: testUUID,
- }
- require.NoError(t, k8sClient.Status().Update(ctx, nb))
-
- require.EventuallyWithT(t, func(t *assert.CollectT) {
- uploadedPathMtx.Lock()
- assert.Equal(t, "/some-signed-url", uploadedPath)
- uploadedPathMtx.Unlock()
- }, timeout, interval, "waiting for command to upload the tarball")
-
- t.Log("wait group waiting")
- wg.Wait()
-}
diff --git a/kubectl/internal/commands/main_test.go b/kubectl/internal/commands/main_test.go
deleted file mode 100644
index 41f55b7c..00000000
--- a/kubectl/internal/commands/main_test.go
+++ /dev/null
@@ -1,146 +0,0 @@
-package commands_test
-
-import (
- "bytes"
- "context"
- "fmt"
- "log"
- "os"
- "path/filepath"
- "testing"
- "time"
-
- ctrl "sigs.k8s.io/controller-runtime"
-
- "k8s.io/client-go/kubernetes"
- "k8s.io/client-go/kubernetes/scheme"
- "k8s.io/client-go/rest"
- "sigs.k8s.io/controller-runtime/pkg/client"
- "sigs.k8s.io/controller-runtime/pkg/envtest"
- logf "sigs.k8s.io/controller-runtime/pkg/log"
- "sigs.k8s.io/controller-runtime/pkg/log/zap"
-
- apiv1 "github.com/substratusai/substratus/api/v1"
- iclient "github.com/substratusai/substratus/kubectl/internal/client"
- "github.com/substratusai/substratus/kubectl/internal/commands"
- //+kubebuilder:scaffold:imports
-)
-
-const (
- timeout = time.Second * 5
- interval = time.Second / 10
- testUUID = "c1d1eb65-75bd-48a5-9bad-802810fc9117"
-)
-
-var (
- kubectlKubeconfigPath string
- k8sClient client.Client
- testEnv *envtest.Environment
- ctx context.Context
- cancel context.CancelFunc
- stdout bytes.Buffer
-)
-
-func TestMain(m *testing.M) {
- commands.NewClient = newClientWithMockPortForward
- commands.NotebookStdout = &stdout
- commands.NewUUID = func() string { return testUUID }
-
- //var buf bytes.Buffer
- logf.SetLogger(zap.New(
- zap.UseDevMode(true),
- //zap.WriteTo(&buf),
- ))
-
- ctx, cancel = context.WithCancel(context.TODO())
-
- log.Println("bootstrapping test environment")
- testEnv = &envtest.Environment{
- CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")},
- ErrorIfCRDPathMissing: true,
- }
-
- cfg, err := testEnv.Start()
- requireNoError(err)
-
- kubectlUser, err := testEnv.ControlPlane.AddUser(envtest.User{
- Name: "kubectl-user",
- Groups: []string{"system:masters"},
- }, nil)
- requireNoError(err)
- kubeconfig, err := kubectlUser.KubeConfig()
- requireNoError(err)
- kubectlKubeconfigPath = filepath.Join(os.TempDir(), "kubeconfig.yaml")
- requireNoError(os.WriteFile(kubectlKubeconfigPath, kubeconfig, 0644))
-
- log.Printf("wrote test kubeconfig to: %s", kubectlKubeconfigPath)
-
- requireNoError(apiv1.AddToScheme(scheme.Scheme))
-
- //+kubebuilder:scaffold:scheme
-
- k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
- requireNoError(err)
-
- mgr, err := ctrl.NewManager(cfg, ctrl.Options{
- Scheme: scheme.Scheme,
- MetricsBindAddress: "0",
- })
- requireNoError(err)
-
- ctx, cancel := context.WithCancel(ctx)
-
- go func() {
- log.Println("starting manager")
- err := mgr.Start(ctx)
- if err != nil {
- log.Printf("starting manager: %s", err)
- }
- }()
-
- log.Println("running tests")
- code := m.Run()
-
- // TODO: Run cleanup on ctrl-C, etc.
- log.Println("stopping manager")
- cancel()
- log.Println("stopping test environment")
- requireNoError(testEnv.Stop())
-
- fmt.Printf(`
-======== Command Stdout ========
-%s
-================================
-`, stdout.String())
-
- os.Exit(code)
-}
-
-func requireNoError(err error) {
- if err != nil {
- log.Fatal(err)
- }
-}
-
-type mockClient struct {
- iclient.Interface
-}
-
-func (c *mockClient) PortForwardNotebook(ctx context.Context, verbose bool, nb *apiv1.Notebook, ready chan struct{}) error {
- log.Println("mockClient.PortForwardNotebook called")
- select {
- case ready <- struct{}{}:
- fmt.Println("sent ready")
- default:
- fmt.Println("no ready sent")
- }
- ctx.Done()
- return fmt.Errorf("mock PortForwardNotebook exiting because of ctx.Done()")
-}
-
-func newClientWithMockPortForward(inf kubernetes.Interface, cfg *rest.Config) iclient.Interface {
- return &mockClient{
- // Only mock PortForwardNotebook()
- Interface: iclient.NewClient(inf, cfg),
- }
-}
diff --git a/kubectl/internal/commands/notebook.go b/kubectl/internal/commands/notebook.go
deleted file mode 100644
index 4e7f60ef..00000000
--- a/kubectl/internal/commands/notebook.go
+++ /dev/null
@@ -1,352 +0,0 @@
-package commands
-
-import (
- "context"
- "errors"
- "flag"
- "fmt"
- "math"
- "os"
- "os/signal"
- "path/filepath"
- "sync"
- "syscall"
- "time"
-
- "github.com/briandowns/spinner"
- "github.com/pkg/browser"
- "github.com/spf13/cobra"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/types"
- "k8s.io/apimachinery/pkg/util/runtime"
- "k8s.io/client-go/kubernetes"
- "k8s.io/client-go/tools/clientcmd"
- "k8s.io/klog/v2"
- "k8s.io/utils/ptr"
-
- apiv1 "github.com/substratusai/substratus/api/v1"
- "github.com/substratusai/substratus/kubectl/internal/client"
-)
-
-func Notebook() *cobra.Command {
- var flags struct {
- dir string
- build string
- kubeconfig string
- filename string
- namespace string
- noOpenBrowser bool
- sync bool
- forceConflicts bool
- noSuspend bool
- timeout time.Duration
- }
-
- cmd := &cobra.Command{
- Use: "notebook [flags] NAME",
- Short: "Start a Jupyter Notebook development environment",
- Args: cobra.MaximumNArgs(1),
- Version: Version,
- RunE: func(cmd *cobra.Command, args []string) error {
- client.Version = Version
-
- ctx, ctxCancel := context.WithCancel(cmd.Context())
- cancel := func() {
- klog.V(1).Info("Context cancelled")
- ctxCancel()
- }
- defer cancel()
-
- // The -v flag is managed by klog, so we need to check it manually.
- var verbose bool
- if cmd.Flag("v").Changed {
- verbose = true
- }
-
- if flags.dir != "" {
- if flags.build == "" {
- flags.build = flags.dir
- }
- // If the user specified a directory, we assume they want to sync
- // unless they explicitly set --sync themselves.
- if !cmd.Flag("sync").Changed {
- flags.sync = true
- }
- // If the user specified a directory, we assume they have a notebook.yaml
- // file in their directory unless they explicitly set --filename themselves.
- if !cmd.Flag("filename").Changed {
- flags.filename = filepath.Join(flags.dir, "notebook.yaml")
- }
- }
-
- sigs := make(chan os.Signal, 1)
- signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
- go func() {
- <-sigs
- cancel()
- }()
-
- spin := spinner.New(spinner.CharSets[9], 100*time.Millisecond)
-
- kubeconfigNamespace, restConfig, err := buildConfigFromFlags("", flags.kubeconfig)
- if err != nil {
- return fmt.Errorf("rest config: %w", err)
- }
-
- namespace := "default"
- if flags.namespace != "" {
- namespace = flags.namespace
- } else if kubeconfigNamespace != "" {
- namespace = kubeconfigNamespace
- }
-
- clientset, err := kubernetes.NewForConfig(restConfig)
- if err != nil {
- return fmt.Errorf("clientset: %w", err)
- }
-
- c := NewClient(clientset, restConfig)
- notebooks, err := c.Resource(&apiv1.Notebook{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "substratus.ai/v1",
- Kind: "Notebook",
- },
- })
- if err != nil {
- return fmt.Errorf("resource client: %w", err)
- }
-
- var obj client.Object
- if len(args) == 1 {
- fetched, err := notebooks.Get(namespace, args[0])
- if err != nil {
- return fmt.Errorf("getting notebook: %w", err)
- }
- obj = fetched.(client.Object)
- } else if flags.filename != "" {
- manifest, err := os.ReadFile(flags.filename)
- if err != nil {
- return fmt.Errorf("reading file: %w", err)
- }
- obj, err = client.Decode(manifest)
- if err != nil {
- return fmt.Errorf("decoding: %w", err)
- }
- if obj.GetNamespace() == "" {
- // When there is no .metadata.namespace set in the manifest...
- obj.SetNamespace(namespace)
- } else {
- // TODO: Closer match kubectl behavior here by differentiaing between
- // the short -n and long --namespace flags.
- // See example kubectl error:
- // error: the namespace from the provided object "a" does not match the namespace "b". You must pass '--namespace=a' to perform this operation.
- if flags.namespace != "" && flags.namespace != obj.GetNamespace() {
- // When there is .metadata.namespace set in the manifest and
- // a conflicting -n or --namespace flag...
- return fmt.Errorf("the namespace from the provided object %q does not match the namespace %q from flag", obj.GetNamespace(), flags.namespace)
- }
- }
- } else {
- return fmt.Errorf("must specify -f (--filename) flag or NAME argument")
- }
-
- nb, err := client.NotebookForObject(obj)
- if err != nil {
- return fmt.Errorf("notebook for object: %w", err)
- }
- nb.Spec.Suspend = ptr.To(false)
-
- var tarball *client.Tarball
- if flags.build != "" {
- spin.Suffix = " Preparing tarball..."
- spin.Start()
-
- var err error
- tarball, err = client.PrepareImageTarball(ctx, flags.build)
- if err != nil {
- return fmt.Errorf("preparing tarball: %w", err)
- }
- defer os.Remove(tarball.TempDir)
-
- spin.Stop()
- fmt.Fprintln(NotebookStdout, "Tarball prepared.")
-
- if err := client.ClearImage(nb); err != nil {
- return fmt.Errorf("clearing image in spec: %w", err)
- }
- if err := client.SetUploadContainerSpec(nb, tarball, NewUUID()); err != nil {
- return fmt.Errorf("setting upload in spec: %w", err)
- }
- }
-
- if err := notebooks.Apply(nb, flags.forceConflicts); err != nil {
- return fmt.Errorf("applying: %w", err)
- }
-
- cleanup := func() {
- // Use a new context to avoid using the cancelled one.
- // ctx := context.Background()
-
- if flags.noSuspend {
- fmt.Fprintln(NotebookStdout, "Skipping notebook suspension, it will keep running.")
- } else {
- // Suspend notebook.
- spin.Suffix = " Suspending notebook..."
- spin.Start()
- _, err := notebooks.Patch(nb.Namespace, nb.Name, types.MergePatchType, []byte(`{"spec": {"suspend": true} }`), &metav1.PatchOptions{})
- spin.Stop()
- if err != nil {
- klog.Errorf("Error suspending notebook: %v", err)
- } else {
- fmt.Fprintln(NotebookStdout, "Notebook suspended.")
- }
- }
- }
- defer cleanup()
-
- if flags.build != "" {
- spin.Suffix = " Uploading tarball..."
- spin.Start()
-
- if err := notebooks.Upload(ctx, nb, tarball); err != nil {
- return fmt.Errorf("uploading: %w", err)
- }
-
- spin.Stop()
- fmt.Fprintln(NotebookStdout, "Tarball uploaded.")
- }
-
- spin.Suffix = " Waiting for Notebook to be ready..."
- spin.Start()
-
- waitReadyCtx, cancelWaitReady := context.WithTimeout(ctx, flags.timeout)
- defer cancelWaitReady() // Avoid context leak.
- if err := notebooks.WaitReady(waitReadyCtx, nb); err != nil {
- return fmt.Errorf("waiting for notebook to be ready: %w", err)
- }
-
- spin.Stop()
- fmt.Fprintln(NotebookStdout, "Notebook ready.")
-
- var wg sync.WaitGroup
-
- if flags.sync {
- wg.Add(1)
- go func() {
- defer func() {
- wg.Done()
- klog.V(2).Info("Syncing files from notebook: Done.")
- // Stop other goroutines.
- cancel()
- }()
- if err := c.SyncFilesFromNotebook(ctx, nb, flags.build); err != nil {
- if !errors.Is(err, context.Canceled) {
- klog.Errorf("Error syncing files from notebook: %v", err)
- }
- }
- }()
- }
-
- serveReady := make(chan struct{})
- wg.Add(1)
- go func() {
- defer func() {
- wg.Done()
- klog.V(2).Info("Port-forwarding: Done.")
- // Stop other goroutines.
- cancel()
- }()
-
- const maxRetries = 3
- for i := 0; i < maxRetries; i++ {
- portFwdCtx, cancelPortFwd := context.WithCancel(ctx)
- defer cancelPortFwd() // Avoid a context leak
- runtime.ErrorHandlers = []func(err error){
- func(err error) {
- // Cancel a broken port forward to attempt to restart the port-forward.
- klog.Errorf("Port-forward error: %v", err)
- cancelPortFwd()
- },
- }
-
- // portForward will close the ready channel when it returns.
- // so we only use the outer ready channel once. On restart of the portForward,
- // we use a new channel.
- var ready chan struct{}
- if i == 0 {
- ready = serveReady
- } else {
- ready = make(chan struct{})
- }
-
- if err := c.PortForwardNotebook(portFwdCtx, verbose, nb, ready); err != nil {
- klog.Errorf("Port-forward returned an error: %v", err)
- }
-
- // Check if the command's context is cancelled, if so,
- // avoid restarting the port forward.
- if err := ctx.Err(); err != nil {
- klog.V(1).Infof("Context done, not attempting to restart port-forward: %v", err.Error())
- return
- }
-
- cancelPortFwd() // Avoid a build up of contexts before returning.
- backoff := time.Duration(math.Pow(2, float64(i))) * time.Second
- klog.V(1).Infof("Restarting port forward (index = %v), after backoff: %s", i, backoff)
- time.Sleep(backoff)
- }
- klog.V(1).Info("Done trying to port-forward")
- }()
-
- spin.Suffix = " Waiting for connection to be ready to serve..."
- spin.Start()
- select {
- case <-serveReady:
- case <-ctx.Done():
- return fmt.Errorf("context done while waiting on connection to be ready: %w", ctx.Err())
- }
- spin.Stop()
- fmt.Fprintln(NotebookStdout, "Connection ready.")
-
- // TODO(nstogner): Grab token from Notebook status.
- url := "http://localhost:8888?token=default"
- if !flags.noOpenBrowser {
- fmt.Fprintf(NotebookStdout, "Opening browser to %s\n", url)
- browser.OpenURL(url)
- } else {
- fmt.Fprintf(NotebookStdout, "Open browser to: %s\n", url)
- }
-
- klog.V(2).Info("Waiting for routines to complete before exiting")
- wg.Wait()
- klog.V(2).Info("Routines completed, exiting")
-
- return nil
- },
- }
-
- defaultKubeconfig := os.Getenv("KUBECONFIG")
- if defaultKubeconfig == "" {
- defaultKubeconfig = clientcmd.RecommendedHomeFile
- }
- cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "")
-
- cmd.Flags().StringVarP(&flags.dir, "dir", "d", "", "Directory to launch the Notebook for. Equivalent to -f /notebook.yaml -b -s")
- cmd.Flags().StringVarP(&flags.build, "build", "b", "", "Build the Notebook from this local directory")
- cmd.Flags().StringVarP(&flags.filename, "filename", "f", "", "Filename identifying the resource to develop against.")
- cmd.Flags().BoolVarP(&flags.sync, "sync", "s", false, "Sync local directory with Notebook")
-
- cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "Namespace of Notebook")
-
- cmd.Flags().BoolVar(&flags.noSuspend, "no-suspend", false, "Do not suspend the Notebook when exiting")
- cmd.Flags().BoolVar(&flags.forceConflicts, "force-conflicts", true, "If true, server-side apply will force the changes against conflicts.")
- cmd.Flags().BoolVar(&flags.noOpenBrowser, "no-open-browser", false, "Do not open the Notebook in a browser")
- cmd.Flags().DurationVarP(&flags.timeout, "timeout", "t", 20*time.Minute, "Timeout for Notebook to become ready")
-
- // Add standard kubectl logging flags (for example: -v=2).
- goflags := flag.NewFlagSet("", flag.PanicOnError)
- klog.InitFlags(goflags)
- cmd.Flags().AddGoFlagSet(goflags)
-
- return cmd
-}
diff --git a/kubectl/internal/commands/notebook_test.go b/kubectl/internal/commands/notebook_test.go
deleted file mode 100644
index 46937097..00000000
--- a/kubectl/internal/commands/notebook_test.go
+++ /dev/null
@@ -1,92 +0,0 @@
-package commands_test
-
-import (
- "context"
- "net/http"
- "net/http/httptest"
- "sync"
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- apiv1 "github.com/substratusai/substratus/api/v1"
- "github.com/substratusai/substratus/kubectl/internal/commands"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/types"
-)
-
-func TestNotebook(t *testing.T) {
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- cmd := commands.Notebook()
- cmd.SetArgs([]string{
- "--filename", "./test-notebook/notebook.yaml",
- "--build", "./test-notebook",
- "--kubeconfig", kubectlKubeconfigPath,
- "--no-open-browser",
- "-v=4",
- })
- cmd.SetContext(ctx)
- var wg sync.WaitGroup
-
- wg.Add(1)
- go func() {
- defer wg.Done()
- if err := cmd.Execute(); err != nil {
- t.Error(err)
- }
- }()
-
- var uploadedPath string
- var uploadedPathMtx sync.Mutex
- mockBucketServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- t.Log("mockBucketServer handler called")
-
- uploadedPathMtx.Lock()
- uploadedPath = r.URL.String()
- uploadedPathMtx.Unlock()
- }))
- defer mockBucketServer.Close()
-
- nb := &apiv1.Notebook{
- ObjectMeta: metav1.ObjectMeta{
- Name: "test-notebook",
- Namespace: "default",
- },
- }
- require.EventuallyWithT(t, func(t *assert.CollectT) {
- err := k8sClient.Get(ctx, types.NamespacedName{Namespace: nb.Namespace, Name: nb.Name}, nb)
- assert.NoError(t, err, "getting notebook")
- }, timeout, interval, "waiting for the notebook to be created")
-
- nb.Status.BuildUpload = apiv1.UploadStatus{
- SignedURL: mockBucketServer.URL + "/some-signed-url",
- RequestID: testUUID,
- }
- require.NoError(t, k8sClient.Status().Update(ctx, nb))
-
- require.EventuallyWithT(t, func(t *assert.CollectT) {
- uploadedPathMtx.Lock()
- assert.Equal(t, "/some-signed-url", uploadedPath)
- uploadedPathMtx.Unlock()
- }, timeout, interval, "waiting for command to upload the tarball")
-
- require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Namespace: nb.Namespace, Name: nb.Name}, nb))
- nb.Status.Ready = true
- require.NoError(t, k8sClient.Status().Update(ctx, nb))
-
- require.EventuallyWithT(t, func(t *assert.CollectT) {
- assert.Contains(t, stdout.String(), "Open browser to")
- }, timeout, interval, "waiting for command to indicate a browser should be opened")
-
- t.Logf("Killing command")
- cancel()
-
- t.Log("Test wait group waiting")
- wg.Wait()
-
- // Use context.Background() because the original context is cancelled.
- require.NoError(t, k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nb.Namespace, Name: nb.Name}, nb))
- require.True(t, nb.IsSuspended(), "Make sure cleanup ran")
-}
diff --git a/kubectl/internal/commands/test-applybuild/Dockerfile b/kubectl/internal/commands/test-applybuild/Dockerfile
deleted file mode 100644
index 64f6dbed..00000000
--- a/kubectl/internal/commands/test-applybuild/Dockerfile
+++ /dev/null
@@ -1,6 +0,0 @@
-FROM substratusai/base
-
-RUN touch touch.txt
-
-COPY . /tmp
-#COPY hello.txt hello.txt
diff --git a/kubectl/internal/commands/test-applybuild/hello.txt b/kubectl/internal/commands/test-applybuild/hello.txt
deleted file mode 100644
index 5ab2f8a4..00000000
--- a/kubectl/internal/commands/test-applybuild/hello.txt
+++ /dev/null
@@ -1 +0,0 @@
-Hello
\ No newline at end of file
diff --git a/kubectl/internal/commands/test-applybuild/notebook.yaml b/kubectl/internal/commands/test-applybuild/notebook.yaml
deleted file mode 100644
index 07bb20b4..00000000
--- a/kubectl/internal/commands/test-applybuild/notebook.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-apiVersion: substratus.ai/v1
-kind: Notebook
-metadata:
- name: test-applybuild
-spec:
- image: substratusai/base:latest
- params:
- foo: bar
- x: 123
diff --git a/kubectl/internal/commands/test-notebook/Dockerfile b/kubectl/internal/commands/test-notebook/Dockerfile
deleted file mode 100644
index 64f6dbed..00000000
--- a/kubectl/internal/commands/test-notebook/Dockerfile
+++ /dev/null
@@ -1,6 +0,0 @@
-FROM substratusai/base
-
-RUN touch touch.txt
-
-COPY . /tmp
-#COPY hello.txt hello.txt
diff --git a/kubectl/internal/commands/test-notebook/hello.txt b/kubectl/internal/commands/test-notebook/hello.txt
deleted file mode 100644
index 5ab2f8a4..00000000
--- a/kubectl/internal/commands/test-notebook/hello.txt
+++ /dev/null
@@ -1 +0,0 @@
-Hello
\ No newline at end of file
diff --git a/kubectl/internal/commands/test-notebook/notebook.yaml b/kubectl/internal/commands/test-notebook/notebook.yaml
deleted file mode 100644
index 742b310d..00000000
--- a/kubectl/internal/commands/test-notebook/notebook.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-apiVersion: substratus.ai/v1
-kind: Notebook
-metadata:
- name: test-notebook
-spec:
- image: substratusai/base:main
- params:
- foo: bar
- x: 123