add healthcheck

.coordinator.env

@@ -26,7 +26,8 @@ TENANT_TOKEN_UN_PROTECTED_TEST=UNPROTECTED
 TENANT_TOKEN_PROTECTED_TEST=jds
 TENANT_TOKEN_RANDOM_TESTING=UNPROTECTED
 
-# The tenant name is specified in the issuing/status invocations like so
+# The tenant name is then specified in 
+# the issuing/status invocations like so
 # (for tenant name econ101):
 # http://myhost.org/instance/econ101/credentials/issue
 # http://myhost.org/instance/econ101/credentials/status

.dockerignore

@@ -1,2 +1,22 @@
 **/*.env
-node_modules
+.git
+.github
+.husky
+coverage
+logs
+node_modules
+.dockerignore
+.editorconfig
+.eslintrc.cjs
+.gitignore
+.lintstagedrc.json
+.prettierignore
+.prettierrc.js
+compose-test.yaml
+compose.yaml
+compose-health-test.yaml
+Dockerfile
+README
+server-dev-only.cert
+server-dev-only.key
+ .env.healthcheck.testing

.gitignore

@@ -108,4 +108,9 @@ dist
 .tern-port
 
 # vscode
-.vscode
+.vscode
+
+compose-test.yaml
+compose.yaml
+compose-health-test.yaml
+.env.healthcheck.testing

healthcheck.js

@@ -0,0 +1,79 @@
+import nodemailer from 'nodemailer'
+import axios from 'axios'
+
+const serviceURL = process.env.HEALTH_CHECK_SERVICE_URL
+const serviceName = process.env.HEALTH_CHECK_SERVICE_NAME
+const shouldPostToWebHook = process.env.HEALTH_CHECK_WEB_HOOK
+const shouldSendEmail =
+  process.env.HEALTH_CHECK_SMTP_HOST &&
+  process.env.HEALTH_CHECK_SMTP_USER &&
+  process.env.HEALTH_CHECK_SMTP_PASS &&
+  process.env.HEALTH_CHECK_EMAIL_FROM &&
+  process.env.HEALTH_CHECK_EMAIL_RECIPIENT
+
+axios
+  .get(serviceURL)
+  .then(async function (response) {
+    try {
+      const body = response.data
+      if (body.healthy === true) {
+        process.exit(0)
+      }
+      await notify(`${serviceName} is unhealthy: ${body.error}`)
+      process.exit(1)
+    } catch (error) {
+      await notify(
+        `${serviceName} is potentially unhealthy - error: ${JSON.stringify(error)}`
+      )
+      process.exit(1)
+    }
+  })
+  .catch(async (error) => {
+    await notify(
+      `${serviceName} is unhealthy and will restart after 3 tries. Error: ${error.message}`
+    )
+    process.exit(1)
+  })
+
+async function notify (message) {
+  console.log(message)
+  if (shouldSendEmail) await sendMail(message)
+  if (shouldPostToWebHook) await postToWebHook(message)
+}
+
+async function postToWebHook (text) {
+  await axios
+    .post(process.env.HEALTH_CHECK_WEB_HOOK, { text })
+    .catch((error) => {
+      console.error(error)
+    })
+}
+
+async function sendMail (message) {
+  const messageParams = {
+    from: process.env.HEALTH_CHECK_EMAIL_FROM,
+    to: process.env.HEALTH_CHECK_EMAIL_RECIPIENT,
+    subject: process.env.HEALTH_CHECK_EMAIL_SUBJECT,
+    text: message
+  }
+
+  const mailTransport = {
+    host: process.env.HEALTH_CHECK_SMTP_HOST,
+    auth: {
+      user: process.env.HEALTH_CHECK_SMTP_USER,
+      pass: process.env.HEALTH_CHECK_SMTP_PASS
+    },
+    ...(process.env.HEALTH_CHECK_SMTP_PORT && {
+      port: process.env.HEALTH_CHECK_SMTP_PORT
+    })
+  }
+
+  const transporter = nodemailer.createTransport(mailTransport)
+
+  try {
+    await transporter.sendMail(messageParams)
+  } catch (error) {
+    console.log('the email send error: ')
+    console.log(error)
+  }
+}

package.json

@@ -7,9 +7,10 @@
     "start": "node -r dotenv/config server.js",
     "dev": "nodemon -r dotenv/config server.js",
     "dev-noenv": "nodemon server.js",
-    "lint": "eslint . --ext .js",
-    "lint-fix": "eslint --fix . --ext .js",
-    "test": "NODE_OPTIONS=--experimental-vm-modules npx mocha --timeout 10000 -r dotenv/config dotenv_config_path=src/test-fixtures/.env.testing src/app.test.js "
+    "lint": "eslint .  --ext .js",
+    "lint-fix": "eslint --fix .  --ext .js",
+    "test": "NODE_OPTIONS=--experimental-vm-modules npx mocha --timeout 10000 -r dotenv/config dotenv_config_path=src/test-fixtures/.env.testing src/app.test.js",
+    "test:health": "node -r dotenv/config healthcheck.js dotenv_config_path=./.env.healthcheck.testing"
   },
   "dependencies": {
     "axios": "^1.4.0",
@@ -18,6 +19,7 @@
     "dotenv": "^16.0.3",
     "express": "~4.16.1",
     "morgan": "~1.9.1",
+    "nodemailer": "^6.9.13",
     "winston": "^3.9.0"
   },
   "devDependencies": {

src/app.js

@@ -6,7 +6,8 @@ import errorHandler from './middleware/errorHandler.js'
 import errorLogger from './middleware/errorLogger.js'
 import invalidPathHandler from './middleware/invalidPathHandler.js'
 import verifyAuthHeader from './verifyAuthHeader.js'
-import { getConfig } from './config.js'
+import { getConfig, defaultTenantName } from './config.js'
+import { getUnsignedVC } from './test-fixtures/vc.js'
 
 class IssuingException extends Error {
   constructor (code, message, error = null) {
@@ -36,6 +37,21 @@ export async function build (opts = {}) {
   app.use(express.urlencoded({ extended: false }))
   app.use(cors())
 
+  app.get('/healthz', async function (req, res) {
+    try {
+      const endpoint = `${req.protocol}://${req.headers.host}/instance/${defaultTenantName}/credentials/issue`
+      const { data } = await axios.post(endpoint, getUnsignedVC())
+      if (!data.proof) { throw new IssuingException(503, 'issuer-coordinator healthz failed') }
+    } catch (e) {
+      console.log(`exception in healthz: ${JSON.stringify(e)}`)
+      return res.status(503).json({
+        error: `issuer-coordinator healthz check failed with error: ${e}`,
+        healthy: false
+      })
+    }
+    res.send({ message: 'issuer-coordinator server status: ok.', healthy: true })
+  })
+
   app.get('/', async function (req, res, next) {
     if (enableStatusService) {
       try {

src/app.test.js

@@ -7,7 +7,7 @@ import protectedNock from './test-fixtures/nocks/protected_status_signing.js'
 import unprotectedStatusUpdateNock from './test-fixtures/nocks/unprotected_status_update.js'
 import unknownStatusIdNock from './test-fixtures/nocks/unknown_status_id_nock.js'
 import protectedStatusUpdateNock from './test-fixtures/nocks/protected_status_update.js'
-
+import healthzStatusSigningNock from './test-fixtures/nocks/healthz_status_signing.js'
 import { build } from './app.js'
 
 let testTenantToken
@@ -227,4 +227,33 @@ describe('api', () => {
       expect(response.status).to.equal(200)
     })
   })
+
+  describe('/healthz', () => {
+    it('returns 200 when healthy', async () => {
+      healthzStatusSigningNock()
+      await request(app)
+        .get('/healthz')
+        .expect('Content-Type', /json/)
+        .expect((res) => {
+          expect(res.body.message).to.contain('ok')
+        })
+        .expect(200)
+    })
+  })
+
+  describe('/healthz fail', () => {
+    // to force an error with the health check, we
+    // simply don't set the nock for the signing and
+    // status services
+
+    it('returns 503 when not healthy', async () => {
+      await request(app)
+        .get('/healthz')
+        .expect('Content-Type', /json/)
+        .expect((res) => {
+          expect(res.body.error).to.contain('error')
+        })
+        .expect(503)
+    })
+  })
 })

src/config.js

@@ -1,6 +1,8 @@
 let CONFIG
 const defaultPort = 4005
-const defaultTenantName = 'test'
+const defaultConsoleLogLevel = 'silly'
+const defaultLogLevel = 'silly'
+export const defaultTenantName = 'test'
 const demoTenantName = 'testing'
 const randomTenantName = 'random'
 const randtomTenantToken = 'UNPROTECTED'
@@ -39,6 +41,10 @@ function parseConfig () {
   const config = Object.freeze({
     enableHttpsForDev: env.ENABLE_HTTPS_FOR_DEV?.toLowerCase() === 'true',
     enableAccessLogging: env.ENABLE_ACCESS_LOGGING?.toLowerCase() === 'true',
+    consoleLogLevel: env.CONSOLE_LOG_LEVEL?.toLocaleLowerCase() || defaultConsoleLogLevel,
+    logLevel: env.LOG_LEVEL?.toLocaleLowerCase() || defaultLogLevel,
+    errorLogFile: env.ERROR_LOG_FILE,
+    logAllFile: env.LOG_ALL_FILE,
     enableStatusService: env.ENABLE_STATUS_SERVICE?.toLowerCase() === 'true',
     signingService: env.SIGNING_SERVICE ?? defaultSigningService,
     statusService: env.STATUS_SERVICE ?? defaultStatusService,