Standalone CLI: sub

Makefile

@@ -189,7 +189,7 @@ dev-down-aws: build-installer
 .PHONY: dev-run-gcp
 # Controller manager configuration #
 dev-run-gcp: export CLOUD=gcp
-dev-run-gcp: PROJECT_ID ?= $(shell gcloud config get project)
+dev-run-gcp: export PROJECT_ID=$(shell gcloud config get project)
 dev-run-gcp: export CLUSTER_NAME=substratus
 dev-run-gcp: export CLUSTER_LOCATION=us-central1
 dev-run-gcp: export PRINCIPAL=substratus@${PROJECT_ID}.iam.gserviceaccount.com

api/v1/conditions.go

@@ -3,9 +3,8 @@ package v1
 const (
 	ConditionUploaded = "Uploaded"
 	ConditionBuilt    = "Built"
-	ConditionLoaded   = "Loaded"
-	ConditionModelled = "Modelled"
-	ConditionDeployed = "Deployed"
+	ConditionComplete = "Complete"
+	ConditionServing  = "Serving"
 )
 
 const (
@@ -20,6 +19,7 @@ const (
 
 	ReasonJobNotComplete     = "JobNotComplete"
 	ReasonJobComplete        = "JobComplete"
+	ReasonJobFailed          = "JobFailed"
 	ReasonDeploymentReady    = "DeploymentReady"
 	ReasonDeploymentNotReady = "DeploymentNotReady"
 	ReasonPodReady           = "PodReady"

api/v1/model_types.go

@@ -23,12 +23,12 @@ type ModelSpec struct {
 	// Resources are the compute resources required by the container.
 	Resources *Resources `json:"resources,omitempty"`
 
-	// BaseModel should be set in order to mount another model to be
+	// Model should be set in order to mount another model to be
 	// used for transfer learning.
-	BaseModel *ObjectRef `json:"baseModel,omitempty"`
+	Model *ObjectRef `json:"model,omitempty"`
 
 	// Dataset to mount for training.
-	TrainingDataset *ObjectRef `json:"trainingDataset,omitempty"`
+	Dataset *ObjectRef `json:"dataset,omitempty"`
 
 	// Parameters are passing into the model training/loading container as environment variables.
 	// Environment variable name will be `"PARAM_" + uppercase(key)`.
@@ -42,6 +42,7 @@ func (m *Model) GetParams() map[string]intstr.IntOrString {
 func (m *Model) GetBuild() *Build {
 	return m.Spec.Build
 }
+
 func (m *Model) SetBuild(b *Build) {
 	m.Spec.Build = b
 }

api/v1/notebook_types.go

@@ -44,6 +44,7 @@ func (n *Notebook) GetParams() map[string]intstr.IntOrString {
 func (n *Notebook) GetBuild() *Build {
 	return n.Spec.Build
 }
+
 func (n *Notebook) SetBuild(b *Build) {
 	n.Spec.Build = b
 }
@@ -79,6 +80,10 @@ func (n *Notebook) GetStatusUpload() UploadStatus {
 	return n.Status.BuildUpload
 }
 
+func (n *Notebook) GetStatusArtifacts() ArtifactsStatus {
+	return n.Status.Artifacts
+}
+
 func (n *Notebook) IsSuspended() bool {
 	return n.Spec.Suspend != nil && *n.Spec.Suspend
 }
@@ -92,6 +97,9 @@ type NotebookStatus struct {
 	// Conditions is the list of conditions that describe the current state of the Notebook.
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
 
+	// Artifacts status.
+	Artifacts ArtifactsStatus `json:"artifacts,omitempty"`
+
 	// BuildUpload contains the status of the build context upload.
 	BuildUpload UploadStatus `json:"buildUpload,omitempty"`
 }

cmd/controllermanager/main.go

@@ -7,17 +7,14 @@ import (
 	"io/ioutil"
 	"os"
 
-	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
-	// to ensure that exec-entrypoint and run can make use of them.
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"gopkg.in/yaml.v2"
-	"k8s.io/client-go/kubernetes"
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
-
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -26,7 +23,6 @@ import (
 	"github.com/substratusai/substratus/internal/cloud"
 	"github.com/substratusai/substratus/internal/controller"
 	"github.com/substratusai/substratus/internal/sci"
-	//+kubebuilder:scaffold:imports
 )
 
 var (

cmd/sub/main.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"log"
+
+	"github.com/substratusai/substratus/internal/cli"
+)
+
+var Version = "development"
+
+func main() {
+	cli.Version = Version
+	if err := cli.Command().Execute(); err != nil {
+		log.Fatal(err)
+	}
+}

config/crd/bases/substratus.ai_models.yaml

@@ -44,16 +44,6 @@ spec:
           spec:
             description: Spec is the desired state of the Model.
             properties:
-              baseModel:
-                description: BaseModel should be set in order to mount another model
-                  to be used for transfer learning.
-                properties:
-                  name:
-                    description: Name of Kubernetes object.
-                    type: string
-                required:
-                - name
-                type: object
               build:
                 description: Build specifies how to build an image.
                 properties:
@@ -113,6 +103,15 @@ spec:
                 items:
                   type: string
                 type: array
+              dataset:
+                description: Dataset to mount for training.
+                properties:
+                  name:
+                    description: Name of Kubernetes object.
+                    type: string
+                required:
+                - name
+                type: object
               env:
                 additionalProperties:
                   type: string
@@ -121,6 +120,16 @@ spec:
               image:
                 description: Image that contains model code and dependencies.
                 type: string
+              model:
+                description: Model should be set in order to mount another model to
+                  be used for transfer learning.
+                properties:
+                  name:
+                    description: Name of Kubernetes object.
+                    type: string
+                required:
+                - name
+                type: object
               params:
                 additionalProperties:
                   anyOf:
@@ -161,15 +170,6 @@ spec:
                     format: int64
                     type: integer
                 type: object
-              trainingDataset:
-                description: Dataset to mount for training.
-                properties:
-                  name:
-                    description: Name of Kubernetes object.
-                    type: string
-                required:
-                - name
-                type: object
             type: object
           status:
             description: Status is the observed state of the Model.

config/crd/bases/substratus.ai_notebooks.yaml

@@ -180,6 +180,12 @@ spec:
           status:
             description: Status is the observed state of the Notebook.
             properties:
+              artifacts:
+                description: Artifacts status.
+                properties:
+                  url:
+                    type: string
+                type: object
               buildUpload:
                 description: BuildUpload contains the status of the build context
                   upload.

containertools/cmd/nbwatch/main.go

@@ -8,14 +8,15 @@ import (
 	"path/filepath"
 	"strings"
 
-	"k8s.io/klog/v2"
-
 	"github.com/fsnotify/fsnotify"
 )
 
 var Version = "development"
 
 func main() {
+	log.SetOutput(os.Stderr)
+	log.Println("Starting")
+
 	if len(os.Args) == 2 && os.Args[1] == "version" {
 		fmt.Printf("nbwatch %s\n", Version)
 		os.Exit(0)
@@ -33,7 +34,31 @@ func run() error {
 	}
 	defer w.Close()
 
-	w.Add("/content/src")
+	const contentDir = "/content"
+
+	// NOTE: Watch is non-recursive.
+	log.Printf("Watching: %v", contentDir)
+	w.Add(contentDir)
+
+	entries, err := os.ReadDir(contentDir)
+	if err != nil {
+		return fmt.Errorf("reading dir: %w", err)
+	}
+	for _, e := range entries {
+		if !e.IsDir() {
+			continue
+		}
+
+		switch name := e.Name(); name {
+		case "data", "model", "artifacts":
+		default:
+			if !strings.HasPrefix(name, ".") {
+				p := filepath.Join(contentDir, name)
+				log.Printf("Watching: %v", p)
+				w.Add(p)
+			}
+		}
+	}
 
 	watchLoop(w)
 
@@ -49,7 +74,7 @@ func watchLoop(w *fsnotify.Watcher) {
 			if !ok { // Channel was closed (i.e. Watcher.Close() was called).
 				return
 			}
-			klog.Error(err)
+			log.Printf("error: %v", err)
 		// Read from Events.
 		case e, ok := <-w.Events:
 			if !ok { // Channel was closed (i.e. Watcher.Close() was called).
@@ -59,9 +84,10 @@ func watchLoop(w *fsnotify.Watcher) {
 			i++
 			path := e.Name
 
+			base := filepath.Base(path)
 			// Covers ".git", ".gitignore", ".gitmodules", ".gitattributes", ".ipynb_checkpoints"
 			// and also temporary files that Jupyter writes on save like: ".~hello.py"
-			if strings.HasPrefix(filepath.Base(path), ".") {
+			if strings.HasPrefix(base, ".") {
 				continue
 			}