Merge pull request #225 from NetsBlox/matlab-timeout

Keep matlab cluster "warm" (prevent worker release)

src/procedures/matlab/keep-warm.js

@@ -0,0 +1,44 @@
+const { setTimeout, setInterval, clearInterval } = require("../../timers");
+const seconds = 1000;
+const minutes = 60 * seconds;
+
+/**
+ * This is made to "keep the matlab cluster warm" and prevent worker nodes from being
+ * released back to the cluster. This class takes an action and, once started, call it
+ * on a given interval for a specified duration.
+ *
+ * Essentially, it is a stateful setInterval w/ an end duration that restarts on each call
+ *
+ * Please use responsibly.
+ */
+class KeepWarm {
+  constructor(action, interval = 10 * seconds) {
+    this.action = action;
+    this.interval = interval;
+    this.currentInterval = null;
+  }
+
+  async keepWarm(duration = 15 * minutes) {
+    this.stop();
+    const intervalId = setInterval(this.action, this.interval);
+    this.currentInterval = intervalId;
+    setTimeout(() => {
+      if (this.currentInterval === intervalId) {
+        this.stop();
+      }
+    }, duration);
+  }
+
+  stop() {
+    if (this.currentInterval) {
+      clearInterval(this.currentInterval);
+      this.currentInterval = null;
+    }
+  }
+
+  isStillWarm() {
+    return !!this.currentInterval;
+  }
+}
+
+module.exports = KeepWarm;

src/procedures/matlab/matlab.js

@@ -10,6 +10,7 @@
 const axios = require("axios");
 
 const { MATLAB_KEY, MATLAB_URL = "" } = process.env;
+const KeepWarm = require("./keep-warm");
 const request = axios.create({
   headers: {
     "X-NetsBlox-Auth-Token": MATLAB_KEY,
@@ -19,6 +20,28 @@ const request = axios.create({
 const MATLAB = {};
 MATLAB.serviceName = "MATLAB";
 
+const warmer = new KeepWarm(async () => {
+  const body = [...new Array(10)].map(() => ({
+    function: "ver",
+    arguments: [],
+    nargout: 1,
+  }));
+  request.post(`${MATLAB_URL}/feval-fast`, body);
+});
+
+async function requestWithRetry(url, body, numRetries = 0) {
+  try {
+    return await request.post(url, body, {
+      timeout: 5000,
+    });
+  } catch (err) {
+    if (err.code === "ECONNABORTED" && numRetries > 0) {
+      return requestWithRetry(url, body, numRetries - 1);
+    }
+    throw err;
+  }
+}
+
 /**
  * Evaluate a MATLAB function with the given arguments and number of return
  * values.
@@ -33,11 +56,15 @@ MATLAB.function = async function (fn, args = [], numReturnValues = 1) {
     arguments: args.map((a) => this._parseArgument(a)),
     nargout: numReturnValues,
   }];
-  // TODO: add timeout detection
-  // TODO: add retry
-  const resp = await request.post(`${MATLAB_URL}/feval-fast`, body, {
-    timeout: 5000,
-  });
+
+  // TODO: if this is the first call, start
+  // high-level alg:
+  //  - start
+  //    - batch requests while starting
+  //    - send requests on start
+  //  - keepWarm
+  const resp = await requestWithRetry(`${MATLAB_URL}/feval-fast`, body, 5);
+  warmer.keepWarm();
   const results = resp.data.FEvalResponse;
   // TODO: add batching queue
   return this._parseResult(results[0]);