first pass on lando 4 service storage

lando · Jul 29, 2024 · 2ce43ca · 2ce43ca
1 parent ba0e1cb
commit 2ce43ca
Show file tree

Hide file tree

Showing 13 changed files with 394 additions and 64 deletions.
diff --git a/app.js b/app.js
@@ -184,6 +184,9 @@ module.exports = async (app, lando) => {
   // Remove meta cache on destroy
   app.events.on('post-destroy', async () => await require('./hooks/app-purge-metadata-cache')(app, lando));
 
+  // Run v4 service destroy methods
+  app.events.on('post-destroy', async () => await require('./hooks/app-run-v4-destroy-service')(app, lando));
+
   // remove v3 build locks
   app.events.on('post-uninstall', async () => await require('./hooks/app-purge-v3-build-locks')(app, lando));
 

diff --git a/builders/lando-v4.js b/builders/lando-v4.js
@@ -116,8 +116,15 @@ module.exports = {
     }
 
     #addRunVolumes(data = []) {
-      // if data is an array then lets concat and uniq to #mounts
-      if (Array.isArray(data)) this.#run.mounts = uniq([...this.#run.mounts, ...data]);
+      // if data is not an array then do nothing
+      if (!Array.isArray(data)) return;
+
+      // run data through normalizeVolumes first so it normalizes our mounts
+      // and then munge it all 2gether
+      this.#run.mounts = uniq([
+        ...this.#run.mounts,
+        ...this.normalizeVolumes(data).map(volume => `${volume.source}:${volume.target}`),
+      ]);
     }
 
     #setupBoot() {
@@ -184,28 +191,31 @@ module.exports = {
       // get this
       super(id, merge({}, {groups}, {states}, upstream), app, lando);
 
-      // foundational this
+      // props
       this.canExec = true;
       this.canHealthcheck = true;
-      this.generateCert = lando.generateCert.bind(lando);
       this.isInteractive = lando.config.isInteractive;
+      this.generateCert = lando.generateCert.bind(lando);
       this.network = lando.config.networkBridge;
       this.project = app.project;
-      this.router = upstream.router;
 
-      // more this
+      // config
       this.certs = config.certs;
       this.command = config.command;
       this.healthcheck = config.healthcheck;
       this.hostnames = uniq([...config.hostnames, `${this.id}.${this.project}.internal`]);
       this.packages = config.packages;
+      this.router = upstream.router;
       this.security = config.security;
       this.security.cas.push(caCert, path.join(path.dirname(caCert), `${caDomain}.pem`));
+      this.storage = require('../utils/normalize-storage')(config.storage, this);
       this.user = user;
 
-      // computed this
-      this.homevol = `${this.project}-${this.user.name}-home`;
-      this.datavol = `${this.project}-${this.id}-data`;
+      // top level stuff
+      this.tlnetworks = {[this.network]: {external: true}};
+      this.tlvolumes = Object.fromEntries(this.storage
+        .filter(volume => volume.type === 'volume')
+        .map(volume => ([volume.name, {external: true}])));
 
       // boot stuff
       this.#setupBoot();
@@ -240,11 +250,9 @@ module.exports = {
       // @TODO: make this into a package?
       this.setNPMRC(lando.config.pluginConfigFile);
 
-      // top level considerations
-      this.addComposeData({
-        networks: {[this.network]: {external: true}},
-        volumes: {[this.homevol]: {}},
-      });
+      // add in top level things
+      this.debug('adding top level volumes %o and networks %o', this.tlvolumes, {networks: this.tlnetworks});
+      this.addComposeData({networks: this.tlnetworks, volumes: this.tlvolumes});
 
       // environment
       const environment = {
@@ -273,6 +281,13 @@ module.exports = {
         'dev.lando.src': app.root,
       }, config.labels);
 
+      // volumes
+      // @TODO: volumes will probably need to handle more than just storage eg mounts?
+      const volumes = this.storage.map(volume => {
+        if (volume.type === 'bind') return volume.mount;
+        return {type: 'volume', source: volume.name, target: volume.destination};
+      });
+
       // add it all 2getha
       this.addLandoServiceData({
         environment,
@@ -281,9 +296,7 @@ module.exports = {
         logging: {driver: 'json-file', options: {'max-file': '3', 'max-size': '10m'}},
         networks: {[this.network]: {aliases: this.hostnames}},
         user: this.user.name,
-        volumes: [
-          `${this.homevol}:/home/${this.user.name}`,
-        ],
+        volumes,
       });
 
       // add any overrides on top
@@ -349,6 +362,7 @@ module.exports = {
     }
 
     // wrapper around addServiceData so we can also add in #run stuff
+    // @TODO: remove user if its set?
     addLandoServiceData(data = {}) {
       // pass through our run considerations
       this.addLandoRunData(data);
@@ -364,6 +378,21 @@ module.exports = {
 
     // buildapp
     async buildApp() {
+      // create storage if needed
+      // @TODO: should this be in try block below?
+      if (this.storage.filter(volume => volume.type === 'volume').length > 0) {
+        const bengine = this.getBengine();
+        await Promise.all(this.storage.filter(volume => volume.type === 'volume').map(async volume => {
+          try {
+            await bengine.createVolume({Name: volume.name, Labels: volume.labels});
+            this.debug('created %o storage volume %o with metadata %o', volume.scope, volume.name, volume.labels);
+          } catch (error) {
+            throw error;
+          }
+        }));
+      }
+
+      // build app
       try {
         // set state
         this.info = {state: {APP: 'BUILDING'}};
@@ -421,6 +450,46 @@ module.exports = {
       return image;
     }
 
+    // remove other app things after a destroy
+    async destroy() {
+      // remove storage if needed
+      if (this.storage.filter(volume => volume.type === 'volume').length > 0) {
+        const bengine = this.getBengine();
+
+        // find the right volumes to trash
+        const {Volumes} = await bengine.listVolumes();
+        const volumes = Volumes
+          .filter(volume => volume?.Labels?.['dev.lando.storage-volume'] === 'TRUE')
+          .map(volume => ({
+            name: volume.Name,
+            scope: volume?.Labels?.['dev.lando.storage-scope'] ?? 'service',
+            project: volume?.Labels?.['dev.lando.storage-project'],
+            service: volume?.Labels?.['dev.lando.storage-service'],
+          }))
+          .filter(volume => volume.project === this.project)
+          .filter(volume => volume.scope === 'service' || volume.scope === 'app')
+          .map(volume => bengine.getVolume(volume.name));
+
+        // and then trash them
+        await Promise.all(volumes.map(async volume => {
+          try {
+            await volume.remove({force: true});
+            this.debug('removed %o volume %o', this.project, volume.name);
+          } catch (error) {
+            throw error;
+          }
+        }));
+      }
+    }
+
+    getBengine() {
+      return LandoServiceV4.getBengine(LandoServiceV4.bengineConfig, {
+        builder: LandoServiceV4.builder,
+        debug: this.debug,
+        orchestrator: LandoServiceV4.orchestrator,
+      });
+    }
+
     async installPackages() {
       await Promise.all(Object.entries(this.packages).map(async ([id, data]) => {
         this.debug('adding package %o with args: %o', id, data);
@@ -440,11 +509,7 @@ module.exports = {
       workingDir = this.appMount,
       entrypoint = ['/bin/bash', '-c'],
     } = {}) {
-      const bengine = LandoServiceV4.getBengine(LandoServiceV4.bengineConfig, {
-        builder: LandoServiceV4.builder,
-        debug: this.debug,
-        orchestrator: LandoServiceV4.orchestrator,
-      });
+      const bengine = this.getBengine();
 
       // construct runopts
       const runOpts = {

diff --git a/components/l337-v4.js b/components/l337-v4.js
@@ -8,6 +8,7 @@ const merge = require('lodash/merge');
 const path = require('path');
 const read = require('../utils/read-file');
 const write = require('../utils/write-file');
+const uniq = require('lodash/uniq');
 
 const {generateDockerFileFromArray} = require('dockerfile-generator/lib/dockerGenerator');
 const {nanoid} = require('nanoid');
@@ -72,6 +73,7 @@ class L337ServiceV4 extends EventEmitter {
         IMAGE: 'UNBUILT',
       },
       steps: [],
+      volumes: [],
     };
   }
 
@@ -233,7 +235,11 @@ class L337ServiceV4 extends EventEmitter {
 
   // just pushes the compose data directly into our thing
   addComposeData(data = {}) {
-    // should we try to consolidate this
+    // if we have a top level volume being added lets add that to #data so we can make use of it in
+    // addServiceData's volume normalization
+    if (data.volumes) this.#data.volumes = uniq([...this.#data.volumes, ...Object.keys(data.volumes)]);
+
+    // @TODO: should we try to consolidate this?
     this.#app.add({
       id: `${this.id}-${nanoid()}`,
       info: this.info,
@@ -384,43 +390,8 @@ class L337ServiceV4 extends EventEmitter {
     const {build, image, ...compose} = data; // eslint-disable-line
 
     // handle any appropriate path normalization for volumes
-    // @TODO: do we need to normalize other things?
     // @NOTE: this normalization ONLY applies here, not in the generic addComposeData
-    if (compose.volumes && Array.isArray(compose.volumes)) {
-      compose.volumes = compose.volumes.map(volume => {
-        // if volume is a one part string then just return so we dont have to handle it downstream
-        if (typeof volume === 'string' && toPosixPath(volume).split(':').length === 1) return volume;
-
-        // if volumes is a string with two colon-separated parts then do stuff
-        if (typeof volume === 'string' && toPosixPath(volume).split(':').length === 2) {
-          const parts = volume.split(':');
-          const target = parts.pop();
-          const source = parts.join(':');
-          volume = {source, target};
-        }
-
-        // if volumes is a string with three colon-separated parts then do stuff
-        if (typeof volume === 'string' && toPosixPath(volume).split(':').length === 3) {
-          const parts = volume.split(':');
-          const mode = parts.pop();
-          const target = parts.pop();
-          const source = parts.join(':');
-          volume = {source, target, read_only: mode === 'ro'};
-        }
-
-        // if source is not an absolute path that exists relateive to appRoot then set as bind
-        if (!path.isAbsolute(volume.source) && fs.existsSync(path.join(this.appRoot, volume.source))) {
-          volume.source = path.join(this.appRoot, volume.source);
-        }
-
-        // we make an "exception" for any /run/host-services things that are in the docker vm
-        if (volume.source.startsWith('/run/host-services')) volume.type = 'bind';
-        else volume.type = fs.existsSync(volume.source) ? 'bind' : 'volume';
-
-        // return
-        return volume;
-      });
-    }
+    if (compose.volumes) compose.volumes = this.normalizeVolumes(compose.volumes);
 
     // add the data
     this.addComposeData({services: {[this.id]: compose}});
@@ -709,6 +680,54 @@ class L337ServiceV4 extends EventEmitter {
     return candidates.length > 0 && candidates[0] !== data ? candidates[0] : false;
   }
 
+  normalizeVolumes(volumes = []) {
+    if (!Array.isArray) return [];
+
+    // normalize and return
+    return volumes.map(volume => {
+      // if volume is a one part string then just return so we dont have to handle it downstream
+      if (typeof volume === 'string' && toPosixPath(volume).split(':').length === 1) return volume;
+
+      // if volumes is a string with two colon-separated parts then do stuff
+      if (typeof volume === 'string' && toPosixPath(volume).split(':').length === 2) {
+        const parts = volume.split(':');
+        const target = parts.pop();
+        const source = parts.join(':');
+        volume = {source, target};
+      }
+
+      // if volumes is a string with three colon-separated parts then do stuff
+      if (typeof volume === 'string' && toPosixPath(volume).split(':').length === 3) {
+        const parts = volume.split(':');
+        const mode = parts.pop();
+        const target = parts.pop();
+        const source = parts.join(':');
+        volume = {source, target, read_only: mode === 'ro'};
+      }
+
+      // at this point we should have an object and if it doesnt have a type we need to try to figure it out
+      // which should be PRETTY straightforward as long as named volumes have been added first
+      if (!volume.type) volume.type = this.#data.volumes.includes(volume.source) ? 'volume' : 'bind';
+
+      // normalize relative bind mount paths to the appRoot
+      if (volume.type === 'bind' && !path.isAbsolute(volume.source)) {
+        volume.source = path.join(this.appRoot, volume.source);
+      }
+
+      // if the bind mount source does not exist then attempt to create it?
+      // we make an "exception" for any /run/host-services things that are in the docker vm
+      // @NOTE: is this actually a good idea?
+      if (volume.type === 'bind'
+        && !fs.existsSync(volume.source)
+        && !volume.source.startsWith('/run/host-services')) {
+        fs.mkdirSync(volume.source, {recursive: true});
+      }
+
+      // return
+      return volume;
+    });
+  }
+
   // sets the base image for the service
   setBaseImage(image, buildArgs = {}) {
     // if the data is raw imagefile instructions then dump it to a file and set to that file

diff --git a/examples/mounts/.lando.yml b/examples/mounts/.lando.yml
@@ -0,0 +1,19 @@
+name: lando-mounts
+services:
+  web3:
+    api: 4
+    image:
+      imagefile: nginxinc/nginx-unprivileged:1.26.1
+      context:
+        - ./default-ssl.conf:/etc/nginx/conf.d/default.conf
+    user: nginx
+    ports:
+      - 8080/http
+      - 8443/https
+
+plugins:
+  "@lando/core": "../.."
+  "@lando/healthcheck": "../../plugins/healthcheck"
+  "@lando/networking": "../../plugins/networking"
+  "@lando/proxy": "../../plugins/proxy"
+  "@lando/scanner": "../../plugins/scanner"
diff --git a/examples/mounts/README.md b/examples/mounts/README.md
@@ -0,0 +1,42 @@
+# Mounts Example
+
+This example exists primarily to test the following documentation:
+
+* [Lando 4 Service Mounts](TBD)
+
+See the [Landofiles](https://docs.lando.dev/config/lando.html) in this directory for the exact magicks.
+
+## Start up tests
+
+```bash
+# Should start
+lando poweroff
+lando start
+```
+
+## Verification commands
+
+Run the following commands to verify things work as expected
+
+```bash
+# Should create a storage volume with app scope by default
+skip
+
+# Should create a storage volume with global scope if specified
+skip
+
+# Should create host bind mounted storage if specified
+# @TODO: relies on TBD MOUNTING system
+skip
+
+# Should remove app scope storage volumes on destroy
+skip
+```
+
+## Destroy tests
+
+```bash
+# Should destroy and poweroff
+lando destroy -y
+lando poweroff
+```
diff --git a/examples/storage/.gitignore b/examples/storage/.gitignore
@@ -0,0 +1 @@
+tmp