Skip to content
This repository has been archived by the owner on Aug 28, 2024. It is now read-only.

Commit

Permalink
Add live-syncing from notebook to local machine (#162)
Browse files Browse the repository at this point in the history
Fixes #140
  • Loading branch information
nstogner authored Aug 3, 2023
1 parent 6725968 commit ddd3606
Show file tree
Hide file tree
Showing 27 changed files with 551 additions and 219 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,6 @@ jobs:
with:
distribution: goreleaser
version: latest
args: release --clean -f kubectl/cmd/notebook/.goreleaser.yaml
- name: Upload assets
uses: actions/upload-artifact@v3
with:
name: kubectl-notebook-plugin
path: |
dist/*
!dist/artifacts.json
!dist/metadata.yaml
!dist/config.yaml
args: release --clean
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
34 changes: 0 additions & 34 deletions .github/workflows/publish-kubectl-applybuild.yaml

This file was deleted.

3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -76,4 +76,5 @@ gcpmanager-skaffold.yaml
gcpmanager-dependencies.yaml
skaffold-dependencies.sh

.ipynb_checkpoints
.ipynb_checkpoints
.vscode/
88 changes: 88 additions & 0 deletions .goreleaser.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
project_name: substratus
before:
hooks:
- go mod tidy
- go generate ./...
release:
prerelease: "true"
builds:
- id: kubectl-applybuild
binary: kubectl-applybuild
main: ./kubectl/cmd/applybuild/
ldflags: "-X 'main.Version={{.Version}}'"
env:
- CGO_ENABLED=0
goos:
- linux
- windows
- darwin
goarch:
- amd64
- arm64
- arm
- id: kubectl-notebook
binary: kubectl-notebook
main: ./kubectl/cmd/notebook/
ldflags: "-X 'main.Version={{.Version}}'"
env:
- CGO_ENABLED=0
goos:
- linux
- windows
- darwin
goarch:
- amd64
- arm64
- arm
- id: containertools-nbwatch
main: ./containertools/cmd/nbwatch/
binary: nbwatch
ldflags: "-X 'main.Version={{.Version}}'"
env:
- CGO_ENABLED=0
goos:
- linux
- windows
- darwin
goarch:
- amd64
- arm64
- arm
archives:
- id: container-tools
builds:
- containertools-nbwatch
format: tar.gz
name_template: >-
container-tools-
{{- .Os }}-
{{- .Arch }}
# use zip for windows archives
format_overrides:
- goos: windows
format: zip
- id: kubectl-plugins
builds:
- kubectl-applybuild
- kubectl-notebook
format: tar.gz
name_template: >-
kubectl-plugins-
{{- .Os }}-
{{- .Arch }}
# use zip for windows archives
format_overrides:
- goos: windows
format: zip
checksum:
name_template: "{{ .ProjectName }}-checksums.txt"
snapshot:
name_template: "{{ incpatch .Version }}-next"
changelog:
sort: asc
filters:
exclude:
- "^docs:"
- "^test:"
# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
# vim: set ts=2 sw=2 tw=0 fo=cnqoj
12 changes: 10 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@

# Image URL to use all building/pushing image targets
VERSION ?= v0.7.0-alpha
VERSION ?= v0.8.0
IMG ?= docker.io/substratusai/controller-manager:${VERSION}
IMG_GCPMANAGER ?= docker.io/substratusai/gcp-manager:${VERSION}

Expand Down Expand Up @@ -216,11 +216,19 @@ install-crds: manifests kustomize ## Install CRDs into the K8s cluster specified
uninstall-crds: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
$(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f -

install/kubernetes/system.yaml: manifests kustomize
.PHONY: installation-scripts
installation-scripts:
perl -pi -e "s/version=.*/version=$(VERSION)/g" install/scripts/install-kubectl-plugins.sh

.PHONY: installation-manifests
installation-manifests: manifests kustomize
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
cd config/gcpmanager && $(KUSTOMIZE) edit set image gcp-manager=${IMG_GCPMANAGER}
$(KUSTOMIZE) build config/default > install/kubernetes/system.yaml

.PHONY: prepare-release
prepare-release: installation-scripts installation-manifests docs

##@ Build Dependencies

## Location to install dependencies to
Expand Down
2 changes: 1 addition & 1 deletion config/gcpmanager/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@ kind: Kustomization
images:
- name: gcp-manager
newName: docker.io/substratusai/gcp-manager
newTag: v0.7.0-alpha
newTag: v0.8.0
2 changes: 1 addition & 1 deletion config/manager/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ kind: Kustomization
images:
- name: controller
newName: docker.io/substratusai/controller-manager
newTag: v0.7.0-alpha
newTag: v0.8.0
- name: gcp-manager
newName: docker.io/substratusai/gcp-manager
newTag: v0.6.5-alpha
83 changes: 83 additions & 0 deletions containertools/cmd/nbwatch/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package main

import (
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"

"k8s.io/klog/v2"

"github.com/fsnotify/fsnotify"
)

var Version = "development"

func main() {
if len(os.Args) == 2 && os.Args[1] == "version" {
fmt.Printf("nbwatch %s\n", Version)
os.Exit(0)
}

if err := run(); err != nil {
log.Fatal(err)
}
}

func run() error {
w, err := fsnotify.NewWatcher()
if err != nil {
return err
}
defer w.Close()

w.Add("/content/src")

watchLoop(w)

return nil
}

func watchLoop(w *fsnotify.Watcher) {
i := int64(0)
for {
select {
// Read from Errors.
case err, ok := <-w.Errors:
if !ok { // Channel was closed (i.e. Watcher.Close() was called).
return
}
klog.Error(err)
// Read from Events.
case e, ok := <-w.Events:
if !ok { // Channel was closed (i.e. Watcher.Close() was called).
return
}

i++
path := e.Name

//path, err := filepath.EvalSymlinks(e.Name)
//if err != nil {
// klog.Error(err)
// continue
//}

switch filepath.Base(path) {
case ".git", ".gitignore", ".gitmodules", ".gitattributes", ".ipynb_checkpoints":
continue
}

encoder.Encode(Event{Index: i, Path: path, Op: e.Op.String()})
}
}
}

var encoder = json.NewEncoder(os.Stdout)

type Event struct {
Index int64 `json:"index"`
Path string `json:"path"`
Op string `json:"op"`
}
18 changes: 17 additions & 1 deletion docs/development.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,22 @@ go build ./kubectl/cmd/notebook && sudo mv notebook /usr/local/bin/kubectl-noteb
go build ./kubectl/cmd/applybuild && sudo mv applybuild /usr/local/bin/kubectl-applybuild
```

The `kubectl notebook` command depends on container-tools for live-syncing. The plugin will try
to download these tools from GitHub releases if they dont already exist with the right versions.

You can build the container-tools for development purposes using the following. NOTE: This is the default cache directory on a mac, this will be different on other machine types.

```sh
export NODE_ARCH=amd64

rm -rf /Users/$USER/Library/Caches/substratus
mkdir -p /Users/$USER/Library/Caches/substratus/container-tools/$NODE_ARCH
GOOS=linux GOARCH=$NODE_ARCH go build ./containertools/cmd/nbwatch
mv nbwatch /Users/$USER/Library/Caches/substratus/container-tools/$NODE_ARCH/

echo "development" > /Users/$USER/Library/Caches/substratus/container-tools/version.txt
```

### Install from release

Release binaries are created for most architectures when the repo is tagged.
Expand All @@ -41,7 +57,7 @@ Be aware that moving the binary to your PATH might fail due to permissions
prompt you for your password:

```sh
bash -c "$(curl -fsSL https://raw.githubusercontent.com/substratusai/substratus/main/install/scripts/install_kubectl_plugin.sh)"
bash -c "$(curl -fsSL https://raw.githubusercontent.com/substratusai/substratus/main/install/scripts/install-kubectl-plugins.sh)"
```

If the plugin installed correctly, you should see it listed as a `kubectl plugin`:
Expand Down
4 changes: 1 addition & 3 deletions examples/notebook/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
FROM substratusai/base

RUN touch touch.txt

COPY hello.txt hello.txt
COPY src src
1 change: 0 additions & 1 deletion examples/notebook/hello.txt

This file was deleted.

3 changes: 1 addition & 2 deletions examples/notebook/src/hello.py
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
print("hello")
#!!!!!!
print("hello!!!")
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ require (
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fatih/color v1.7.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/fsnotify/fsnotify v1.6.0
github.com/go-logr/zapr v1.2.4 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.1 // indirect
Expand Down
4 changes: 2 additions & 2 deletions install/kubernetes/system.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1587,7 +1587,7 @@ spec:
envFrom:
- configMapRef:
name: system
image: docker.io/substratusai/controller-manager:v0.7.0-alpha
image: docker.io/substratusai/controller-manager:v0.8.0
livenessProbe:
httpGet:
path: /healthz
Expand Down Expand Up @@ -1634,7 +1634,7 @@ spec:
app: gcp-manager
spec:
containers:
- image: docker.io/substratusai/gcp-manager:v0.7.0-alpha
- image: docker.io/substratusai/gcp-manager:v0.8.0
imagePullPolicy: Always
livenessProbe:
failureThreshold: 3
Expand Down
15 changes: 15 additions & 0 deletions install/scripts/install-kubectl-plugins.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env bash
set -xe

version=v0.8.0
os=$(uname -s)
arch=$(uname -m | sed 's/aarch64/arm64/g' | sed 's/x86_64/amd64/g')

# NOTE: The URL does not mind if you pass os="Darwin" or os="darwin".
release_url="https://github.com/substratusai/substratus/releases/download/$version/kubectl-plugins-$os-$arch.tar.gz"

wget -qO- $release_url | tar zxv --directory /tmp
chmod +x /tmp/kubectl-applybuild
chmod +x /tmp/kubectl-notebook
mv /tmp/kubectl-applybuild /usr/local/bin/ || sudo mv /tmp/kubectl-applybuild /usr/local/bin/
mv /tmp/kubectl-notebook /usr/local/bin/ || sudo mv /tmp/kubectl-notebook /usr/local/bin/
19 changes: 0 additions & 19 deletions install/scripts/install_kubectl_plugin.sh

This file was deleted.

Loading

0 comments on commit ddd3606

Please sign in to comment.