[spore-drive] fix displacing keys & ts client #264 (#265)

.github/workflows/pre-commit.yml

@@ -21,7 +21,7 @@ jobs:
     - uses: actions/setup-python@v3
     - uses: pre-commit/[email protected]
 
-  tests:
+  go-tests:
     runs-on: ubuntu-latest
     needs: pre-commit
     services:
@@ -63,6 +63,35 @@ jobs:
           fi
           sleep 5
         done
+
+  js-tests:
+    runs-on: ubuntu-latest
+    needs: pre-commit
+    services:
+      docker:
+        image: docker:19.03.12
+        options: --privileged # required for nested containers
+        ports:
+          - 2375:2375
+        volumes:
+          - /var/lib/docker
+    steps:
+    - name: Set up Docker environment
+      run: |
+        docker info
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.22.0'
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Add entry to /etc/hosts
+      run: |
+        echo "127.0.0.1 hal.computers.com" | sudo tee -a /etc/hosts
+    - name: Build mock server
+      run: |
+        cd pkg/spore-drive/clients/mock
+        go build .
     - name: Run JS/TS tests with retries
       run: |
         max_attempts=5

pkg/spore-drive/clients/js/index.ts

@@ -8,6 +8,7 @@ import {
   TauVersion,
   TauUrl,
   TauPath,
+  Course,
 } from "./src/Drive";
 
 class Config extends BaseConfig {
@@ -27,8 +28,8 @@ class Config extends BaseConfig {
 class Drive extends BaseDrive {
   private service: Service;
 
-  constructor(config: Config) {
-    super(config);
+  constructor(config: Config, tau?: TauBinarySource) {
+    super(config, tau);
     this.service = new Service();
   }
 
@@ -41,11 +42,12 @@ class Drive extends BaseDrive {
 export {
   Config,
   Drive,
+  Course,
   CourseConfig,
   TauBinarySource,
   TauLatest,
   TauVersion,
   TauUrl,
   TauPath,
-  Service
+  Service,
 };

pkg/spore-drive/clients/js/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@taubyte/spore-drive",
-  "version": "0.1.1",
-  "service": "0.1.0",
+  "version": "0.1.2",
+  "service": "0.1.1",
   "description": "",
   "type": "module",
   "main": "dist/index.js",

pkg/spore-drive/clients/js/src/Config.ts

@@ -1,3 +1,4 @@
+import { stringify } from "querystring";
 import {
   Source,
   Op,
@@ -101,9 +102,14 @@ class BaseOperation {
       const pathItem = this.path[i];
       const caseName = pathItem.case;
       const messageValue: any = { op };
+
       if (pathItem.name) {
         messageValue.name = pathItem.name;
       }
+      if (pathItem.shape) {
+        messageValue.shape = pathItem.shape;
+      }
+
       op = { case: caseName, value: messageValue };
     }
     return op;
@@ -258,9 +264,9 @@ class Bootstrap extends BaseOperation {
   }
 
   Shape(shapeName: string) {
-    return new BootstrapShape(this.client, this.config, [
+    return new BootstrapShape(this.client,this.config, [
       ...this.path,
-      { case: "select", name: shapeName },
+      { case: "select", shape: shapeName},
     ]);
   }
 

pkg/spore-drive/clients/js/src/Drive.ts

@@ -82,7 +82,7 @@ export class CourseConfig {
   }
 }
 
-class Course {
+export class Course {
   private client: RPCClient;
   private drive: DriveMessage;
   private course?: CourseMessage;

pkg/spore-drive/clients/js/src/Service.test.ts

@@ -5,6 +5,7 @@ describe("Service", () => {
 
   beforeAll(() => {
     service = new Service();
+    service["packageVersion"]="0.1.0" // override to a published version
   });
 
   afterAll(async () => {

pkg/spore-drive/drive/displace.go

@@ -24,7 +24,7 @@ func (d *sporedrive) Displace(ctx context.Context, course course.Course) <-chan
 
 	hyphae := course.Hyphae()
 
-	pCh := make(chan Progress, hyphae.Size()*64)
+	pCh := make(chan Progress, hyphae.Size()*1024)
 
 	if cap(pCh) == 0 { // course is empty
 		close(pCh)
@@ -144,8 +144,61 @@ func (d *sporedrive) uploadPlugins(ctx context.Context, h remoteHost, dir string
 	return nil
 }
 
+func (d *sporedrive) writeSwarmKeyToTmp(h remoteHost) error {
+	skr, err := d.parser.Cloud().P2P().Swarm().Open()
+	if err != nil {
+		return fmt.Errorf("failed to open swarm key: %w", err)
+	}
+	defer skr.Close()
+
+	skf, err := h.OpenFile("/tmp/swarm.key", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600)
+	if err != nil {
+		return fmt.Errorf("failed to open /tmp/swarm.key: %w", err)
+	}
+	defer skf.Close()
+
+	_, err = io.Copy(skf, skr)
+
+	return err
+}
+
+func (d *sporedrive) writeDomainPrivKeyToTmp(h remoteHost) error {
+	pkr, err := d.parser.Cloud().Domain().Validation().OpenPrivateKey()
+	if err != nil {
+		return fmt.Errorf("failed to open private domain key: %w", err)
+	}
+	defer pkr.Close()
+
+	pkf, err := h.OpenFile("/tmp/dv_private.key", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600)
+	if err != nil {
+		return fmt.Errorf("failed to open /tmp/dv_private.key: %w", err)
+	}
+	defer pkf.Close()
+
+	_, err = io.Copy(pkf, pkr)
+
+	return err
+}
+
+func (d *sporedrive) writeDomainPubKeyToTmp(h remoteHost) error {
+	pkr, err := d.parser.Cloud().Domain().Validation().OpenPrivateKey()
+	if err != nil {
+		return fmt.Errorf("failed to open private domain key: %w", err)
+	}
+	defer pkr.Close()
+
+	pkf, err := h.OpenFile("/tmp/dv_public.key", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600)
+	if err != nil {
+		return fmt.Errorf("failed to open /tmp/dv_public.key: %w", err)
+	}
+	defer pkf.Close()
+
+	_, err = io.Copy(pkf, pkr)
+
+	return err
+}
+
 func (d *sporedrive) writeConfig(h remoteHost, shape string, w io.Writer) error {
-	// TODO
 	hc := d.parser.Hosts().Host(h.Host().Name())
 	hshape := hc.Shapes().Instance(shape)
 	sc := d.parser.Shapes().Shape(shape)
@@ -185,7 +238,7 @@ func (d *sporedrive) writeConfig(h remoteHost, shape string, w io.Writer) error
 						return fmt.Errorf("`%s` is not valid IP or CIDR", addr)
 					}
 				}
-				bootstrap = append(bootstrap, fmt.Sprintf("/ip4/%s/tcp/%d/%s", ip.String(), shMainPort, shid))
+				bootstrap = append(bootstrap, fmt.Sprintf("/ip4/%s/tcp/%d/p2p/%s", ip.String(), shMainPort, shid))
 			}
 		}
 
@@ -238,6 +291,18 @@ func (d *sporedrive) uploadConfig(h remoteHost, shape string) error {
 		return err
 	}
 
+	if err = d.writeSwarmKeyToTmp(h); err != nil {
+		return err
+	}
+
+	if err = d.writeDomainPrivKeyToTmp(h); err != nil {
+		return err
+	}
+
+	if err = d.writeDomainPubKeyToTmp(h); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -457,20 +522,6 @@ func (d *sporedrive) displaceHandler(hypha *course.Hypha, progressCh chan<- Prog
 			pushProgress("upload tau", 100)
 		}
 
-		if updatingTau {
-			pushProgress("upload tau files", 0)
-			// upload systemd and other files
-			if err = d.uploadSystemdFile(r); err != nil {
-				return pushError("upload tau files", fmt.Errorf("failed to upload tau files to /tmp: %w", err))
-			}
-
-			if _, err = r.Sudo(ctx, "cp", "-f", "/tmp/[email protected]", "/lib/systemd/system/[email protected]"); err != nil {
-				return pushError("upload tau files", fmt.Errorf("failed to copy [email protected]: %w", err))
-			}
-
-			pushProgress("upload tau files", 100)
-		}
-
 		pushProgress("upload plugins", 0)
 		if err = d.uploadPlugins(ctx, r, "/tmp/tau-plugins"); err != nil {
 			return pushError("upload plugins", fmt.Errorf("failed to upload tau plugins to /tmp: %w", err))
@@ -502,19 +553,51 @@ func (d *sporedrive) displaceHandler(hypha *course.Hypha, progressCh chan<- Prog
 		}
 		pushProgress("setup tau", 20)
 
+		if updatingTau {
+			// upload systemd and other files
+			if err = d.uploadSystemdFile(r); err != nil {
+				return pushError("upload tau files", fmt.Errorf("failed to upload tau files to /tmp: %w", err))
+			}
+
+			if _, err = r.Sudo(ctx, "cp", "-f", "/tmp/[email protected]", "/lib/systemd/system/[email protected]"); err != nil {
+				return pushError("upload tau files", fmt.Errorf("failed to copy [email protected]: %w", err))
+			}
+		}
+
+		if _, err = r.Sudo(ctx, "bash", "-c", "mkdir -p /tb/{bin,scripts,priv,cache,logs,storage,config/keys,plugins}"); err != nil {
+			return pushError("setup tau", fmt.Errorf("failed create fs structure: %w", err))
+		}
+
+		pushProgress("setup tau", 25)
+
 		for _, shape := range hypha.Shapes {
 			if !slices.Contains(hshapes, shape) {
 				continue
 			}
 
 			// upload config
-			d.uploadConfig(r, shape)
+			if err := d.uploadConfig(r, shape); err != nil {
+				return pushError("setup tau", fmt.Errorf("failed to create %s.yaml: %w", shape, err))
+			}
 
 			if _, err = r.Sudo(ctx, "cp", "-f", "/tmp/"+shape+".yaml", "/tb/config/"); err != nil {
 				return pushError("setup tau", fmt.Errorf("failed to copy %s.yaml: %w", shape, err))
 			}
+
+			if _, err = r.Sudo(ctx, "cp", "-f", "/tmp/swarm.key", "/tb/config/keys/"); err != nil {
+				return pushError("setup tau", fmt.Errorf("failed to copy swarm.key: %w", err))
+			}
+
+			if _, err = r.Sudo(ctx, "cp", "-f", "/tmp/dv_private.key", "/tb/config/keys/"); err != nil {
+				return pushError("setup tau", fmt.Errorf("failed to copy dv_private.key: %w", err))
+			}
+
+			if _, err = r.Sudo(ctx, "cp", "-f", "/tmp/dv_public.key", "/tb/config/keys/"); err != nil {
+				return pushError("setup tau", fmt.Errorf("failed to copy dv_public.key: %w", err))
+			}
 		}
 		pushProgress("setup tau", 30)
+
 		if _, err = r.Sudo(ctx, "cp", "-f", "/tmp/tau", "/tb/bin/"); err != nil {
 			return pushError("setup tau", fmt.Errorf("failed to copy tau: %w", err))
 		}

pkg/spore-drive/drive/displace_test.go

@@ -85,7 +85,22 @@ func testDisplace(t *testing.T, sd Spore) {
 			rh.On("Sudo", ctx, "cp", "-f", "/tmp/[email protected]", "/lib/systemd/system/[email protected]").Return(nil, nil)
 		}
 
+		rh.On("Sudo", ctx, "bash", "-c", "mkdir -p /tb/{bin,scripts,priv,cache,logs,storage,config/keys,plugins}").Return(nil, nil)
+
 		// setup tau
+		swarmk, _ := fses[h].OpenFile("/tmp/swarm.key", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600)
+		rh.On("OpenFile", "/tmp/swarm.key", os.O_CREATE|os.O_RDWR|os.O_TRUNC, fs.FileMode(0600)).Once().Return(swarmk, nil)
+
+		dprivk, _ := fses[h].OpenFile("/tmp/dv_private.key", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600)
+		rh.On("OpenFile", "/tmp/dv_private.key", os.O_CREATE|os.O_RDWR|os.O_TRUNC, fs.FileMode(0600)).Once().Return(dprivk, nil)
+
+		dpubk, _ := fses[h].OpenFile("/tmp/dv_public.key", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600)
+		rh.On("OpenFile", "/tmp/dv_public.key", os.O_CREATE|os.O_RDWR|os.O_TRUNC, fs.FileMode(0600)).Once().Return(dpubk, nil)
+
+		rh.On("Sudo", ctx, "cp", "-f", "/tmp/swarm.key", "/tb/config/keys/").Return(nil, nil)
+		rh.On("Sudo", ctx, "cp", "-f", "/tmp/dv_private.key", "/tb/config/keys/").Return(nil, nil)
+		rh.On("Sudo", ctx, "cp", "-f", "/tmp/dv_public.key", "/tb/config/keys/").Return(nil, nil)
+
 		sh1cf, _ := fses[h].OpenFile("/tmp/shape1.yaml", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0750)
 		rh.On("OpenFile", "/tmp/shape1.yaml", os.O_CREATE|os.O_RDWR|os.O_TRUNC, fs.FileMode(0750)).Once().Return(sh1cf, nil)
 		rh.On("Sudo", ctx, "cp", "-f", "/tmp/shape1.yaml", "/tb/config/").Return(nil, nil)
@@ -143,9 +158,9 @@ func testDisplace(t *testing.T, sd Spore) {
 	}
 
 	if updatingTau {
-		assert.Equal(t, len(steps), 52)
+		assert.Equal(t, len(steps), 50)
 	} else {
-		assert.Equal(t, len(steps), 42)
+		assert.Equal(t, len(steps), 44)
 	}
 
 	for h, mfs := range fses {

pkg/spore-drive/examples/js/basic/src/index.ts

@@ -1,4 +1,4 @@
-import {Config,CourseConfig,Drive} from "@taubyte/spore-drive";
+import {Config,CourseConfig,Drive,TauLatest} from "@taubyte/spore-drive";
 
 export const createConfig = async (config: Config) => {
     // Set Cloud Domain
@@ -60,7 +60,7 @@ await config.init()
 
 await createConfig(config)
 
-const drive:Drive = new Drive(config)
+const drive:Drive = new Drive(config,TauLatest)
 
 await drive.init()
 
@@ -71,4 +71,7 @@ await course.displace()
 console.log("displacement...")
 for await (const prg of await course.progress()) {
     console.log(prg)
-}
+}
+
+
+console.log("done")