diff --git a/.github/workflows/publish-kubectl-notebook.yaml b/.github/workflows/goreleaser.yaml similarity index 60% rename from .github/workflows/publish-kubectl-notebook.yaml rename to .github/workflows/goreleaser.yaml index c66629e5..17426551 100644 --- a/.github/workflows/publish-kubectl-notebook.yaml +++ b/.github/workflows/goreleaser.yaml @@ -20,15 +20,6 @@ jobs: with: distribution: goreleaser version: latest - args: release --clean -f kubectl/cmd/notebook/.goreleaser.yaml - - name: Upload assets - uses: actions/upload-artifact@v3 - with: - name: kubectl-notebook-plugin - path: | - dist/* - !dist/artifacts.json - !dist/metadata.yaml - !dist/config.yaml + args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish-kubectl-applybuild.yaml b/.github/workflows/publish-kubectl-applybuild.yaml deleted file mode 100644 index b4849d23..00000000 --- a/.github/workflows/publish-kubectl-applybuild.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: goreleaser -on: - push: - tags: - - "*" -permissions: - contents: write -jobs: - goreleaser: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v4 - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v4 - with: - distribution: goreleaser - version: latest - args: release --clean -f kubectl/cmd/applybuild/.goreleaser.yaml - - name: Upload assets - uses: actions/upload-artifact@v3 - with: - name: kubectl-applybuild-plugin - path: | - dist/* - !dist/artifacts.json - !dist/metadata.yaml - !dist/config.yaml - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index ae04e266..06f85911 100644 --- a/.gitignore +++ b/.gitignore @@ -76,4 +76,5 @@ gcpmanager-skaffold.yaml gcpmanager-dependencies.yaml skaffold-dependencies.sh -.ipynb_checkpoints \ No newline at end of file +.ipynb_checkpoints +.vscode/ \ No newline at end of file diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 00000000..c6255626 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,88 @@ +project_name: substratus +before: + hooks: + - go mod tidy + - go generate ./... +release: + prerelease: "true" +builds: + - id: kubectl-applybuild + binary: kubectl-applybuild + main: ./kubectl/cmd/applybuild/ + ldflags: "-X 'main.Version={{.Version}}'" + env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + goarch: + - amd64 + - arm64 + - arm + - id: kubectl-notebook + binary: kubectl-notebook + main: ./kubectl/cmd/notebook/ + ldflags: "-X 'main.Version={{.Version}}'" + env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + goarch: + - amd64 + - arm64 + - arm + - id: containertools-nbwatch + main: ./containertools/cmd/nbwatch/ + binary: nbwatch + ldflags: "-X 'main.Version={{.Version}}'" + env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + goarch: + - amd64 + - arm64 + - arm +archives: + - id: container-tools + builds: + - containertools-nbwatch + format: tar.gz + name_template: >- + container-tools- + {{- .Os }}- + {{- .Arch }} + # use zip for windows archives + format_overrides: + - goos: windows + format: zip + - id: kubectl-plugins + builds: + - kubectl-applybuild + - kubectl-notebook + format: tar.gz + name_template: >- + kubectl-plugins- + {{- .Os }}- + {{- .Arch }} + # use zip for windows archives + format_overrides: + - goos: windows + format: zip +checksum: + name_template: "{{ .ProjectName }}-checksums.txt" +snapshot: + name_template: "{{ incpatch .Version }}-next" +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +# vim: set ts=2 sw=2 tw=0 fo=cnqoj diff --git a/Makefile b/Makefile index a7f693f3..c233cd78 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Image URL to use all building/pushing image targets -VERSION ?= v0.7.0-alpha +VERSION ?= v0.8.0 IMG ?= docker.io/substratusai/controller-manager:${VERSION} IMG_GCPMANAGER ?= docker.io/substratusai/gcp-manager:${VERSION} @@ -216,11 +216,19 @@ install-crds: manifests kustomize ## Install CRDs into the K8s cluster specified uninstall-crds: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - -install/kubernetes/system.yaml: manifests kustomize +.PHONY: installation-scripts +installation-scripts: + perl -pi -e "s/version=.*/version=$(VERSION)/g" install/scripts/install-kubectl-plugins.sh + +.PHONY: installation-manifests +installation-manifests: manifests kustomize cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} cd config/gcpmanager && $(KUSTOMIZE) edit set image gcp-manager=${IMG_GCPMANAGER} $(KUSTOMIZE) build config/default > install/kubernetes/system.yaml +.PHONY: prepare-release +prepare-release: installation-scripts installation-manifests docs + ##@ Build Dependencies ## Location to install dependencies to diff --git a/config/gcpmanager/kustomization.yaml b/config/gcpmanager/kustomization.yaml index 3ee62547..1395632f 100644 --- a/config/gcpmanager/kustomization.yaml +++ b/config/gcpmanager/kustomization.yaml @@ -6,4 +6,4 @@ kind: Kustomization images: - name: gcp-manager newName: docker.io/substratusai/gcp-manager - newTag: v0.7.0-alpha + newTag: v0.8.0 diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 69af03cf..156d9158 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -5,7 +5,7 @@ kind: Kustomization images: - name: controller newName: docker.io/substratusai/controller-manager - newTag: v0.7.0-alpha + newTag: v0.8.0 - name: gcp-manager newName: docker.io/substratusai/gcp-manager newTag: v0.6.5-alpha diff --git a/containertools/cmd/nbwatch/main.go b/containertools/cmd/nbwatch/main.go new file mode 100644 index 00000000..64b3ff9d --- /dev/null +++ b/containertools/cmd/nbwatch/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + + "k8s.io/klog/v2" + + "github.com/fsnotify/fsnotify" +) + +var Version = "development" + +func main() { + if len(os.Args) == 2 && os.Args[1] == "version" { + fmt.Printf("nbwatch %s\n", Version) + os.Exit(0) + } + + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { + w, err := fsnotify.NewWatcher() + if err != nil { + return err + } + defer w.Close() + + w.Add("/content/src") + + watchLoop(w) + + return nil +} + +func watchLoop(w *fsnotify.Watcher) { + i := int64(0) + for { + select { + // Read from Errors. + case err, ok := <-w.Errors: + if !ok { // Channel was closed (i.e. Watcher.Close() was called). + return + } + klog.Error(err) + // Read from Events. + case e, ok := <-w.Events: + if !ok { // Channel was closed (i.e. Watcher.Close() was called). + return + } + + i++ + path := e.Name + + //path, err := filepath.EvalSymlinks(e.Name) + //if err != nil { + // klog.Error(err) + // continue + //} + + switch filepath.Base(path) { + case ".git", ".gitignore", ".gitmodules", ".gitattributes", ".ipynb_checkpoints": + continue + } + + encoder.Encode(Event{Index: i, Path: path, Op: e.Op.String()}) + } + } +} + +var encoder = json.NewEncoder(os.Stdout) + +type Event struct { + Index int64 `json:"index"` + Path string `json:"path"` + Op string `json:"op"` +} diff --git a/docs/development.md b/docs/development.md index b446a1c3..bcdb9c25 100644 --- a/docs/development.md +++ b/docs/development.md @@ -33,6 +33,22 @@ go build ./kubectl/cmd/notebook && sudo mv notebook /usr/local/bin/kubectl-noteb go build ./kubectl/cmd/applybuild && sudo mv applybuild /usr/local/bin/kubectl-applybuild ``` +The `kubectl notebook` command depends on container-tools for live-syncing. The plugin will try +to download these tools from GitHub releases if they dont already exist with the right versions. + +You can build the container-tools for development purposes using the following. NOTE: This is the default cache directory on a mac, this will be different on other machine types. + +```sh +export NODE_ARCH=amd64 + +rm -rf /Users/$USER/Library/Caches/substratus +mkdir -p /Users/$USER/Library/Caches/substratus/container-tools/$NODE_ARCH +GOOS=linux GOARCH=$NODE_ARCH go build ./containertools/cmd/nbwatch +mv nbwatch /Users/$USER/Library/Caches/substratus/container-tools/$NODE_ARCH/ + +echo "development" > /Users/$USER/Library/Caches/substratus/container-tools/version.txt +``` + ### Install from release Release binaries are created for most architectures when the repo is tagged. @@ -41,7 +57,7 @@ Be aware that moving the binary to your PATH might fail due to permissions prompt you for your password: ```sh -bash -c "$(curl -fsSL https://raw.githubusercontent.com/substratusai/substratus/main/install/scripts/install_kubectl_plugin.sh)" +bash -c "$(curl -fsSL https://raw.githubusercontent.com/substratusai/substratus/main/install/scripts/install-kubectl-plugins.sh)" ``` If the plugin installed correctly, you should see it listed as a `kubectl plugin`: diff --git a/examples/notebook/Dockerfile b/examples/notebook/Dockerfile index 83bc6161..afb60a06 100644 --- a/examples/notebook/Dockerfile +++ b/examples/notebook/Dockerfile @@ -1,5 +1,3 @@ FROM substratusai/base -RUN touch touch.txt - -COPY hello.txt hello.txt +COPY src src diff --git a/examples/notebook/hello.txt b/examples/notebook/hello.txt deleted file mode 100644 index d76f0287..00000000 --- a/examples/notebook/hello.txt +++ /dev/null @@ -1 +0,0 @@ -Hello there!!! \ No newline at end of file diff --git a/examples/notebook/src/hello.py b/examples/notebook/src/hello.py index c253cbc1..8c35c8ee 100644 --- a/examples/notebook/src/hello.py +++ b/examples/notebook/src/hello.py @@ -1,2 +1 @@ -print("hello") -#!!!!!! \ No newline at end of file +print("hello!!!") diff --git a/go.mod b/go.mod index 89dc8bc9..88f449b3 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fatih/color v1.7.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect diff --git a/install/kubernetes/system.yaml b/install/kubernetes/system.yaml index 13b72dd3..f5231d31 100644 --- a/install/kubernetes/system.yaml +++ b/install/kubernetes/system.yaml @@ -1587,7 +1587,7 @@ spec: envFrom: - configMapRef: name: system - image: docker.io/substratusai/controller-manager:v0.7.0-alpha + image: docker.io/substratusai/controller-manager:v0.8.0 livenessProbe: httpGet: path: /healthz @@ -1634,7 +1634,7 @@ spec: app: gcp-manager spec: containers: - - image: docker.io/substratusai/gcp-manager:v0.7.0-alpha + - image: docker.io/substratusai/gcp-manager:v0.8.0 imagePullPolicy: Always livenessProbe: failureThreshold: 3 diff --git a/install/scripts/install-kubectl-plugins.sh b/install/scripts/install-kubectl-plugins.sh new file mode 100755 index 00000000..55fc7fd4 --- /dev/null +++ b/install/scripts/install-kubectl-plugins.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -xe + +version=v0.8.0 +os=$(uname -s) +arch=$(uname -m | sed 's/aarch64/arm64/g' | sed 's/x86_64/amd64/g') + +# NOTE: The URL does not mind if you pass os="Darwin" or os="darwin". +release_url="https://github.com/substratusai/substratus/releases/download/$version/kubectl-plugins-$os-$arch.tar.gz" + +wget -qO- $release_url | tar zxv --directory /tmp +chmod +x /tmp/kubectl-applybuild +chmod +x /tmp/kubectl-notebook +mv /tmp/kubectl-applybuild /usr/local/bin/ || sudo mv /tmp/kubectl-applybuild /usr/local/bin/ +mv /tmp/kubectl-notebook /usr/local/bin/ || sudo mv /tmp/kubectl-notebook /usr/local/bin/ diff --git a/install/scripts/install_kubectl_plugin.sh b/install/scripts/install_kubectl_plugin.sh deleted file mode 100755 index ab483554..00000000 --- a/install/scripts/install_kubectl_plugin.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -xe - -REPO='https://github.com/substratusai/substratus' -LATEST_RELEASE=$(curl ${REPO}/releases -s | - grep 'Link--primary' | - head -n1 | - perl -n -e '/v([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9]+)?)/ && print $&') -OS=$(uname -s) -ARCH=$(uname -m | sed 's/aarch64/arm64/g') -LATEST_OPEN_NOTEBOOK_ARTIFACT_URL=$(echo $LATEST_RELEASE | awk -v repo=$REPO -v os=$OS -v arch=$ARCH -v release=$LATEST_RELEASE '{print repo "/releases/download/" release "/kubectl-open-notebook_" os "_" arch ".tar.gz"}') -LATEST_BUILD_REMOTE_ARTIFACT_URL=$(echo $LATEST_RELEASE | awk -v repo=$REPO -v os=$OS -v arch=$ARCH -v release=$LATEST_RELEASE '{print repo "/releases/download/" release "/kubectl-build-remote_" os "_" arch ".tar.gz"}') - -wget -qO- ${LATEST_OPEN_NOTEBOOK_ARTIFACT_URL} | tar zxv -wget -qO- ${LATEST_BUILD_REMOTE_ARTIFACT_URL} | tar zxv -chmod +x kubectl-open-notebook -chmod +x kubectl-build-remote -mv kubectl-open-notebook /usr/local/bin/ || sudo mv kubectl-open-notebook /usr/local/bin/ -mv kubectl-build-remote /usr/local/bin/ || sudo mv kubectl-build-remote /usr/local/bin/ diff --git a/kubectl/cmd/applybuild/.goreleaser.yaml b/kubectl/cmd/applybuild/.goreleaser.yaml deleted file mode 100644 index daf1c062..00000000 --- a/kubectl/cmd/applybuild/.goreleaser.yaml +++ /dev/null @@ -1,46 +0,0 @@ -project_name: kubectl-applybuild -before: - hooks: - - go mod tidy - - go generate ./... -release: - prerelease: "true" -builds: - - main: ./kubectl/cmd/applybuild/ - env: - - CGO_ENABLED=0 - goos: - - linux - - windows - - darwin - goarch: - - amd64 - - arm64 - - arm - - ppc64le -archives: - - id: archive - format: tar.gz - name_template: >- - {{ .ProjectName }}_ - {{- title .Os }}_ - {{- if eq .Arch "amd64" }}x86_64 - {{- else if eq .Arch "386" }}i386 - {{- else }}{{ .Arch }}{{ end }} - {{- if .Arm }}v{{ .Arm }}{{ end }} - # use zip for windows archives - format_overrides: - - goos: windows - format: zip -checksum: - name_template: "{{ .ProjectName }}-checksums.txt" -snapshot: - name_template: "{{ incpatch .Version }}-next" -changelog: - sort: asc - filters: - exclude: - - "^docs:" - - "^test:" -# yaml-language-server: $schema=https://goreleaser.com/static/schema.json -# vim: set ts=2 sw=2 tw=0 fo=cnqoj diff --git a/kubectl/cmd/applybuild/main.go b/kubectl/cmd/applybuild/main.go index 686c5b2e..93c2dd38 100644 --- a/kubectl/cmd/applybuild/main.go +++ b/kubectl/cmd/applybuild/main.go @@ -5,7 +5,10 @@ import ( "k8s.io/klog/v2" ) +var Version = "development" + func main() { + commands.Version = Version if err := commands.ApplyBuild().Execute(); err != nil { klog.Fatal(err) } diff --git a/kubectl/cmd/notebook/.goreleaser.yaml b/kubectl/cmd/notebook/.goreleaser.yaml deleted file mode 100644 index bd32d054..00000000 --- a/kubectl/cmd/notebook/.goreleaser.yaml +++ /dev/null @@ -1,46 +0,0 @@ -project_name: kubectl-notebook -before: - hooks: - - go mod tidy - - go generate ./... -release: - prerelease: "true" -builds: - - main: ./kubectl/cmd/notebook/ - env: - - CGO_ENABLED=0 - goos: - - linux - - windows - - darwin - goarch: - - amd64 - - arm64 - - arm - - ppc64le -archives: - - id: archive - format: tar.gz - name_template: >- - {{ .ProjectName }}_ - {{- title .Os }}_ - {{- if eq .Arch "amd64" }}x86_64 - {{- else if eq .Arch "386" }}i386 - {{- else }}{{ .Arch }}{{ end }} - {{- if .Arm }}v{{ .Arm }}{{ end }} - # use zip for windows archives - format_overrides: - - goos: windows - format: zip -checksum: - name_template: "{{ .ProjectName }}-checksums.txt" -snapshot: - name_template: "{{ incpatch .Version }}-next" -changelog: - sort: asc - filters: - exclude: - - "^docs:" - - "^test:" -# yaml-language-server: $schema=https://goreleaser.com/static/schema.json -# vim: set ts=2 sw=2 tw=0 fo=cnqoj diff --git a/kubectl/cmd/notebook/main.go b/kubectl/cmd/notebook/main.go index c1dce6db..293be1f6 100644 --- a/kubectl/cmd/notebook/main.go +++ b/kubectl/cmd/notebook/main.go @@ -5,7 +5,10 @@ import ( "k8s.io/klog/v2" ) +var Version = "development" + func main() { + commands.Version = Version if err := commands.Notebook().Execute(); err != nil { klog.Fatal(err) } diff --git a/kubectl/internal/client/client.go b/kubectl/internal/client/client.go index ae904705..68963bb4 100644 --- a/kubectl/internal/client/client.go +++ b/kubectl/internal/client/client.go @@ -18,6 +18,8 @@ import ( apiv1 "github.com/substratusai/substratus/api/v1" ) +var Version = "development" + type Object = client.Object var FieldManager = "kubectl" @@ -31,6 +33,7 @@ var _ Interface = &Client{} type Interface interface { PortForwardNotebook(ctx context.Context, verbose bool, nb *apiv1.Notebook, ready chan struct{}) error Resource(obj Object) (*Resource, error) + SyncFilesFromNotebook(context.Context, *apiv1.Notebook) error } func NewClient(inf kubernetes.Interface, cfg *rest.Config) Interface { diff --git a/kubectl/internal/client/sync.go b/kubectl/internal/client/sync.go index d0c197c7..e114aa67 100644 --- a/kubectl/internal/client/sync.go +++ b/kubectl/internal/client/sync.go @@ -1,17 +1,271 @@ package client import ( + "archive/tar" + "bufio" + "compress/gzip" "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" "path/filepath" + "strings" + "sync" apiv1 "github.com/substratusai/substratus/api/v1" "github.com/substratusai/substratus/kubectl/internal/cp" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/klog/v2" ) -func CopySrcToNotebook(ctx context.Context, baseDir string, nb *apiv1.Notebook) error { - return cp.ToPod(ctx, filepath.Join(baseDir, "src"), "/content/", podForNotebook(nb)) +func (c *Client) SyncFilesFromNotebook(ctx context.Context, nb *apiv1.Notebook) error { + podRef := podForNotebook(nb) + const containerName = "notebook" + + cacheDir, err := os.UserCacheDir() + if err != nil { + return fmt.Errorf("determining user cache directory: %w", err) + } + toolsPath := filepath.Join(cacheDir, "substratus", "container-tools") + if err := os.MkdirAll(toolsPath, 0755); err != nil { + return fmt.Errorf("creating cache directory: %w", err) + } + + const ( + // Assuming linux Nodes. + targetOS = "linux" + ) + + nodeArch, err := c.getNodeArchForPod(ctx, podRef.Name, podRef.Namespace) + if err != nil { + return fmt.Errorf("getting node arch: %w", err) + } + + if err := getContainerTools(toolsPath, targetOS); err != nil { + return fmt.Errorf("getting container-tools: %w", err) + } + + // TODO: Download nbwatch if it doesn't exist. + + nbWatchPath := filepath.Join(toolsPath, nodeArch, "nbwatch") + if err := cp.ToPod(ctx, nbWatchPath, "/tmp/nbwatch", podRef, containerName); err != nil { + return fmt.Errorf("cp nbwatch to pod: %w", err) + } + + r, w := io.Pipe() + + // TODO: Instead of processing events line-by-line, decode them line-by-line + // immediately and append them to a channel with deduplication. + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer func() { + wg.Done() + klog.V(2).Info("File sync loop: Done.") + }() + + klog.V(2).Info("Reading events...") + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + eventLine := scanner.Bytes() + var event NBWatchEvent + if err := json.Unmarshal(eventLine, &event); err != nil { + klog.Errorf("Failed to unmarshal nbevent: %w", err) + } + + relPath, err := filepath.Rel("/content/src", event.Path) + if err != nil { + klog.Errorf("Failed to determining relative path: %w", err) + continue + } + + localDir := "src" + localPath := filepath.Join(localDir, relPath) + + // Possible: CREATE, REMOVE, WRITE, RENAME, CHMOD + if event.Op == "WRITE" || event.Op == "CREATE" { + // NOTE: A long-running port-forward might be more performant here. + if err := cp.FromPod(ctx, event.Path, localPath, podRef, containerName); err != nil { + klog.Errorf("Sync: failed to copy: %w", err) + } + } else if event.Op == "REMOVE" || event.Op == "RENAME" { + if err := os.Remove(localPath); err != nil { + klog.Errorf("Sync: failed to remove: %w", err) + } + } + } + if err := scanner.Err(); err != nil { + klog.Error("Error reading from buffer:", err) + return + } + klog.V(2).Info("Done reading events.") + }() + + if err := c.exec(ctx, podRef, "/tmp/nbwatch", nil, w, os.Stderr); err != nil { + return fmt.Errorf("exec: nbwatch: %w", err) + } + + klog.V(2).Info("Waiting for file sync loop to finish...") + wg.Wait() + + return nil +} + +func (c *Client) exec(ctx context.Context, podRef types.NamespacedName, + command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + cmd := []string{ + "sh", + "-c", + command, + } + req := c.Interface.CoreV1().RESTClient().Post().Resource("pods").Name(podRef.Name). + Namespace(podRef.Namespace).SubResource("exec") + option := &corev1.PodExecOptions{ + Command: cmd, + Stdin: true, + Stdout: true, + Stderr: true, + TTY: true, + } + if stdin == nil { + option.Stdin = false + } + req.VersionedParams( + option, + scheme.ParameterCodec, + ) + exec, err := remotecommand.NewSPDYExecutor(c.Config, "POST", req.URL()) + if err != nil { + return err + } + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + }) + if err != nil { + return err + } + + return nil +} + +type NBWatchEvent struct { + Index int64 `json:"index"` + Path string `json:"path"` + Op string `json:"op"` +} + +func getContainerTools(dir, targetOS string) error { + // Check to see if tools need to be downloaded. + versionPath := filepath.Join(dir, "version.txt") + exists, err := fileExists(versionPath) + if err != nil { + return fmt.Errorf("checking if version file exists: %w", err) + } + if exists { + version, err := os.ReadFile(versionPath) + if err != nil { + return fmt.Errorf("reading version file: %w", err) + } + versionStr := strings.TrimSpace(string(version)) + if versionStr == Version { + klog.V(1).Infof("Version (%q) matches for container-tools, skipping download.", Version) + return nil + } else { + klog.V(1).Infof("Version (%q) does not match version.txt: %q", Version, versionStr) + } + } + + // Remove existing files. + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("removing existing files: %w", err) + } + + for _, arch := range []string{"amd64", "arm64"} { + archDir := filepath.Join(dir, arch) + if err := os.MkdirAll(archDir, 0755); err != nil { + return fmt.Errorf("recreating directory: %w", err) + } + if err := getContainerToolsRelease(archDir, targetOS, arch); err != nil { + return fmt.Errorf("getting container-tools: %w", err) + } + } + + if err := os.WriteFile(versionPath, []byte(Version), 0644); err != nil { + return fmt.Errorf("writing version file: %w", err) + } + + return nil } -func CopySrcFromNotebook(ctx context.Context, baseDir string, nb *apiv1.Notebook) error { - return cp.FromPod(ctx, "/content/src", filepath.Join(baseDir, "src"), podForNotebook(nb)) +func getContainerToolsRelease(dir, targetOS, targetArch string) error { + releaseURL := fmt.Sprintf("https://github.com/substratusai/substratus/releases/download/v%s/container-tools-%s-%s.tar.gz", Version, targetOS, targetArch) + klog.V(1).Infof("Downloading: %s", releaseURL) + resp, err := http.Get(releaseURL) + if err != nil { + return fmt.Errorf("downloading release: %w", err) + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("downloading release: %s", resp.Status) + } + defer resp.Body.Close() + + gzr, err := gzip.NewReader(resp.Body) + if err != nil { + return fmt.Errorf("creating gzip reader: %w", err) + } + tr := tar.NewReader(gzr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break // End of archive + } + if err != nil { + return fmt.Errorf("reading tar: %w", err) + } + + dest := filepath.Join(dir, hdr.Name) + klog.V(1).Infof("Writing %s", dest) + f, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)) + if err != nil { + return fmt.Errorf("creating file: %w", err) + } + if _, err := io.Copy(f, tr); err != nil { + return fmt.Errorf("writing file from tar: %w", err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("closing file: %w", err) + } + } + + return nil +} + +func (c *Client) getNodeArchForPod(ctx context.Context, podName, podNamespace string) (string, error) { + pod, err := c.Interface.CoreV1().Pods(podNamespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("getting pod: %w", err) + } + nodeName := pod.Spec.NodeName + + node, err := c.Interface.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("getting node: %w", err) + } + + arch, ok := node.Labels["kubernetes.io/arch"] + if !ok { + return "", fmt.Errorf("node %s has no kubernetes.io/arch label", nodeName) + } + + return arch, nil } diff --git a/kubectl/internal/client/upload.go b/kubectl/internal/client/upload.go index 7a34443c..c0d331a6 100644 --- a/kubectl/internal/client/upload.go +++ b/kubectl/internal/client/upload.go @@ -38,7 +38,11 @@ type Tarball struct { } func PrepareImageTarball(buildPath string) (*Tarball, error) { - if !fileExists(filepath.Join(buildPath, "Dockerfile")) { + exists, err := fileExists(filepath.Join(buildPath, "Dockerfile")) + if err != nil { + return nil, fmt.Errorf("checking if Dockerfile exists: %w", err) + } + if !exists { return nil, fmt.Errorf("path does not contain Dockerfile: %s", buildPath) } @@ -267,12 +271,15 @@ func tarGz(src, dst string) error { }) } -func fileExists(filename string) bool { +func fileExists(filename string) (bool, error) { info, err := os.Stat(filename) - if os.IsNotExist(err) { - return false + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err } - return !info.IsDir() + return !info.IsDir(), nil } func uploadTarball(tarball *Tarball, url string) error { diff --git a/kubectl/internal/commands/applybuild.go b/kubectl/internal/commands/applybuild.go index cee4ce9d..740c45d1 100644 --- a/kubectl/internal/commands/applybuild.go +++ b/kubectl/internal/commands/applybuild.go @@ -22,11 +22,15 @@ func ApplyBuild() *cobra.Command { } var cmd = &cobra.Command{ - Use: "applybuild [flags] BUILD_CONTEXT", - Args: cobra.ExactArgs(1), - Short: "Apply a Substratus object, upload and build container in-cluster from a local directory", + Use: "applybuild [flags] BUILD_CONTEXT", + Args: cobra.ExactArgs(1), + Short: "Apply a Substratus object, upload and build container in-cluster from a local directory", + Version: Version, RunE: func(cmd *cobra.Command, args []string) error { + client.Version = Version + ctx := cmd.Context() + if cfg.filename == "" { return fmt.Errorf("-f (--filename) is required") } diff --git a/kubectl/internal/commands/notebook.go b/kubectl/internal/commands/notebook.go index cb5d57d2..53b6c44c 100644 --- a/kubectl/internal/commands/notebook.go +++ b/kubectl/internal/commands/notebook.go @@ -2,6 +2,7 @@ package commands import ( "context" + "errors" "flag" "fmt" "os" @@ -40,11 +41,21 @@ func Notebook() *cobra.Command { } var cmd = &cobra.Command{ - Use: "notebook [flags] NAME", - Short: "Start a Jupyter Notebook development environment", - Args: cobra.MaximumNArgs(1), + Use: "notebook [flags] NAME", + Short: "Start a Jupyter Notebook development environment", + Args: cobra.MaximumNArgs(1), + Version: Version, RunE: func(cmd *cobra.Command, args []string) error { + client.Version = Version + ctx, cancel := context.WithCancel(cmd.Context()) + defer cancel() + + // The -v flag is managed by klog, so we need to check it manually. + var verbose bool + if cmd.Flag("v").Changed { + verbose = true + } if cfg.dir != "" { if cfg.build == "" { @@ -153,21 +164,7 @@ func Notebook() *cobra.Command { cleanup := func() { // Use a new context to avoid using the cancelled one. - ctx := context.Background() - - if cfg.sync { - spin.Suffix = " Syncing notebook to local directory..." - spin.Start() - baseDir := "." - if cfg.build != "" { - baseDir = cfg.build - } - if err := client.CopySrcFromNotebook(ctx, baseDir, nb); err != nil { - klog.Errorf("Error syncing notebook to local directory: %v", err) - } - spin.Stop() - fmt.Fprintln(NotebookStdout, "Synced notebook src/ to local directory.") - } + //ctx := context.Background() if cfg.noSuspend { fmt.Fprintln(NotebookStdout, "Skipping notebook suspension, it will keep running.") @@ -208,26 +205,32 @@ func Notebook() *cobra.Command { spin.Stop() fmt.Fprintln(NotebookStdout, "Notebook ready.") + var wg sync.WaitGroup + if cfg.sync { - spin.Suffix = " Syncing local directory with Notebook..." - spin.Start() - baseDir := "." - if cfg.build != "" { - baseDir = cfg.build - } - if err := client.CopySrcToNotebook(ctx, baseDir, nb); err != nil { - return fmt.Errorf("copying src to notebook: %w", err) - } - spin.Stop() - fmt.Fprintln(NotebookStdout, "Synced src/ to notebook.") + wg.Add(1) + go func() { + defer func() { + wg.Done() + klog.V(2).Info("Syncing files from notebook: Done.") + + }() + if err := c.SyncFilesFromNotebook(ctx, nb); err != nil { + if !errors.Is(err, context.Canceled) { + klog.Errorf("Error syncing files from notebook: %v", err) + } + cancel() + } + }() } - var wg sync.WaitGroup - serveReady := make(chan struct{}) wg.Add(1) go func() { - defer wg.Done() + defer func() { + wg.Done() + klog.V(2).Info("Port-forwarding: Done.") + }() first := true for { @@ -251,7 +254,7 @@ func Notebook() *cobra.Command { ready = make(chan struct{}) } - if err := c.PortForwardNotebook(portFwdCtx, true, nb, ready); err != nil { + if err := c.PortForwardNotebook(portFwdCtx, verbose, nb, ready); err != nil { klog.Errorf("Port-forward returned an error: %v", err) return } diff --git a/kubectl/internal/commands/utils.go b/kubectl/internal/commands/utils.go index e3398444..81550c66 100644 --- a/kubectl/internal/commands/utils.go +++ b/kubectl/internal/commands/utils.go @@ -8,6 +8,8 @@ import ( "github.com/substratusai/substratus/kubectl/internal/client" ) +var Version = "development" + // NewClient is a dirty hack to allow the client to be mocked out in tests. var NewClient = client.NewClient diff --git a/kubectl/internal/cp/kubectl.go b/kubectl/internal/cp/kubectl.go index 0a8f87e6..344d7fde 100644 --- a/kubectl/internal/cp/kubectl.go +++ b/kubectl/internal/cp/kubectl.go @@ -12,14 +12,14 @@ import ( "k8s.io/apimachinery/pkg/types" ) -func ToPod(ctx context.Context, src, dst string, pod types.NamespacedName) error { - cmd := exec.CommandContext(ctx, "kubectl", "cp", "-n", pod.Namespace, "-c", "notebook", src, pod.Name+":"+dst) +func ToPod(ctx context.Context, src, dst string, pod types.NamespacedName, container string) error { + cmd := exec.CommandContext(ctx, "kubectl", "cp", "-n", pod.Namespace, "-c", container, src, pod.Name+":"+dst) cmd.Stderr = os.Stderr return cmd.Run() } -func FromPod(ctx context.Context, src, dst string, pod types.NamespacedName) error { - cmd := exec.CommandContext(ctx, "kubectl", "cp", "-n", pod.Namespace, "-c", "notebook", pod.Name+":"+src, dst) +func FromPod(ctx context.Context, src, dst string, pod types.NamespacedName, container string) error { + cmd := exec.CommandContext(ctx, "kubectl", "cp", "-n", pod.Namespace, "-c", container, pod.Name+":"+src, dst) cmd.Stderr = os.Stderr return cmd.Run() }