feat: add api endpoint to generate and serve cached file (#1)

* feat: add API endpoint to return a cached file

* feat: compress response

* feat: define precise error codes

* chore: rename file to reflect its purpose

* fix: fix missing new line in CSV file

* fix: fix choices columns for votes with multiple choices

* refactor: avoid passing `proposal` object

* fix: use different name for incomplete file

* fix: prevent race condition on file generation

* feat: use AWS as cached file storage

* feat: separate cache fetcher and generator in API endpoint

* fix: do not return an error code on cache generation success

* feat: add additional file storage engine

For easier tests

* chore: update README

* fix: use inferred env variables for S3 client setup

* feat: custimize location of cached files

* feat: protect generate endpoint behind authentication

* chore: add LICENCE file

* feat: add webhook support for `generate` endpoint

* feat: add basic queue system to handle cache file generation

Queue system enable back the cache file generation on demand when cache is not available on fetch

* chore: update README

* chore: add tests

* chore: fix github worflow

* fix: fix folder creation

* chore: rename github workflow

* chore: update README

* chorel; add test for weighted votes

* feat: add reason to votes report

* chore: fix false positives in linter

* chore: dependencies upgrade

* chore: fix CI job name

* chore: typecheck task should not also build

* refactor: always throw Error objects instead of literal

* chore: set minimum required node version

* Update readme

* Minor updates

* fix: add default route

* fix: use await syntax

* chore: remove unecessary rule

* refactor: set the storage engine via an env variable

* refactor: keep private method to bottom of files

* refactor: set the StorageEngine and its related setting to environment variables

* chore: lint task should fix by default

For consistency with other projects

---------

Co-authored-by: ChaituVR <[email protected]>

.env.example

@@ -1 +1,8 @@
 HUB_URL=https://hub.snapshot.org
+AWS_ACCESS_KEY_ID=
+AWS_REGION=
+AWS_SECRET_ACCESS_KEY=
+AWS_BUCKET_NAME=
+WEBHOOK_AUTH_TOKEN=
+STORAGE_ENGINE=file
+VOTE_REPORT_SUBDIR=votes

.github/workflows/lint.yml

@@ -1,7 +1,7 @@
-name: Build and Run Tests
+name: Run Lint
 on: [push]
 jobs:
-  build-test:
+  build-lint:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
@@ -11,5 +11,5 @@ jobs:
           cache: 'yarn'
       - run: yarn install
       - run: yarn build
-      - run: yarn lint
+      - run: yarn lint:nofix
       - run: yarn typecheck

.github/workflows/test.yml

@@ -0,0 +1,14 @@
+name: Build and Run Tests
+on: [push]
+jobs:
+  build-test:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+          cache: 'yarn'
+      - run: yarn install
+      - run: echo -en "HUB_URL=https://hub.snapshot.org" > .env
+      - run: yarn test

.gitignore

@@ -4,6 +4,7 @@ dist
 build
 .env
 coverage
+tmp
 
 # Remove some common IDE working directories
 .idea

LICENCE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) Snapshot Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,55 +1,129 @@
 # Snapshot/Sidekick
 
-Sidekick is the service serving all proposals' votes CSV report
+Sidekick is the service serving all proposal's votes CSV report
 
-<hr>
+---
 
 This service is exposing an API endpoint expecting a closed proposal ID, and will
 return a CSV file with all the given proposal's votes.
 
-NOTE: CSV files are generated only once, then cached, making this service a cache middleware
-for snapshot-hub, for proposals' votes.
+> NOTE: CSV files are generated only once, then cached, making this service a cache middleware between snapshot-hub and UI
 
 ## Project Setup
 
+### Requirements
+
+node >= 18.0.0
+
 ### Dependencies
 
 Install the dependencies
 
-```
+```bash
 yarn
 ```
 
-_This project does not require a database, but may need larger server storage capacity for the cached files_
+_This project does not require a database, but requires a [storage engine](#storage-engine)_
 
 ### Configuration
 
-Edit the hub API url in the `.env` file if needed
+Copy `.env.example`, rename it to `.env` and edit the hub API url in the `.env` file if needed
 
-```
+```bash
 HUB_URL=https://hub.snapshot.org
 ```
 
-## Compiles and hot-reloads for development
+If you are using AWS as storage engine, set all the required `AWS_` config keys, and set `STORAGE_ENGINE` to `aws`.
 
-```
+### Storage engine
+
+This script is shipped with 2 storage engine.
+
+You can set the cache engine by toggling the `STORAGE_ENGINE` environment variable.
+
+| `STORAGE_ENGINE` | Description | Cache save path                   |
+| ---------------- | ----------- | --------------------------------- |
+| `aws`            | Amazon S3   | `public/`                         |
+| `file` (default) | Local file  | `tmp/` (relative to project root) |
+
+You can additionally specify a sub directory by setting `VOTE_REPORT_SUBDIR`
+(By default, all votes report will be nested in the `votes` directory).
+
+### Compiles and hot-reloads for development
+
+```bash
 yarn dev
 ```
 
-## Linting and typecheck
+## Linting, typecheck and test
 
-```
+```bash
 yarn lint
 yarn typecheck
+yarn test
 ```
 
-## Build for production
+## Usage
+
+Retrieving and generating the cache file have their own respective endpoint
+
+### Fetch a cache file
+
+Send a POST request with a proposal ID
 
+```bash
+curl -X POST localhost:3000/votes/[PROPOSAL-ID]
 ```
+
+When cached, this request will respond with a stream to a CSV file.
+
+On all other cases, it will respond with a [JSON-RPC 2.0](https://www.jsonrpc.org/specification) error response:
+
+```bash
+{
+  "jsonrpc":"2.0",
+  "error":{
+    "code": CODE,
+    "message": MESSAGE
+  },
+  "id": PROPOSAL-ID
+}
+```
+
+| Description                         | `CODE` | `MESSAGE`           |
+| ----------------------------------- | ------ | ------------------- |
+| When the proposal does not exist    | -40001 | PROPOSAL_NOT_FOUND  |
+| When the proposal is not closed     | -40004 | PROPOSAL_NOT_CLOSED |
+| When the file is pending generation | -40010 | PENDING_GENERATION  |
+| Other/Unknown/Server Error          | -32603 | INTERNAL_ERROR      |
+
+Furthermore, when votes report can be cached, but does not exist yet, a cache generation task will be queued. This enable cache to be generated on-demand.
+
+### Generate a cache file
+
+Send a POST request with a body following the [Webhook event object](https://docs.snapshot.org/tools/webhooks).
+
+```bash
+curl -X POST localhost:3000/votes/generate \
+-H "Authenticate: WEBHOOK_AUTH_TOKEN" \
+-H "Content-Type: application/json" \
+-d '{"id": "proposal/[PROPOSAL-ID]", "event": "proposal/end"}'
+```
+
+- On success, will respond with a success [JSON-RPC 2.0](https://www.jsonrpc.org/specification) message
+- On error, will respond with the same result and codes as the `fetch` endpoint above
+
+The endpoint has been designed to receive events from snapshot webhook service.
+
+Do not forget to set `WEBHOOK_AUTH_TOKEN` in the `.env` file
+
+## Build for production
+
+```bash
 yarn build
 yarn start
 ```
 
 ## License
 
-[MIT](https://github.com/snapshot-labs/envelop-ui/blob/bootstrap-app/LICENSE)
+[MIT](https://github.com/snapshot-labs/snapshot-sidekick/blob/main/LICENCE)

jest.config.ts

@@ -0,0 +1,20 @@
+/*
+ * For a detailed explanation regarding each configuration property and type check, visit:
+ * https://jestjs.io/docs/configuration
+ */
+
+export default {
+  clearMocks: true,
+  collectCoverage: true,
+  coverageDirectory: 'coverage',
+  coverageProvider: 'v8',
+
+  // An array of regexp pattern strings used to skip coverage collection
+  coveragePathIgnorePatterns: ['/node_modules/', '<rootDir>/dist/', '<rootDir>/test/fixtures/'],
+
+  preset: 'ts-jest',
+  testEnvironment: 'jest-environment-node-single-context',
+  setupFiles: ['dotenv/config'],
+  //setupFilesAfterEnv: ['<rootDir>src/setupTests.ts'],
+  moduleFileExtensions: ['js', 'ts']
+};

package.json

@@ -5,32 +5,47 @@
   "repository": "snapshot-labs/snapshot-sidekick",
   "license": "MIT",
   "scripts": {
-    "lint": "eslint src/ --ext .ts",
-    "lint:fix": "eslint src/ --ext .ts --fix",
+    "lint:nofix": "eslint src/ --ext .ts",
+    "lint": "yarn lint:nofix --fix",
     "build": "tsc -p tsconfig.json",
     "dev": "nodemon src/index.ts -e js,ts",
     "start": "node dist/src/index.js",
-    "typecheck": "tsc"
+    "typecheck": "tsc --noEmit",
+    "test": "jest"
   },
   "eslintConfig": {
     "extends": "@snapshot-labs"
   },
+  "engines": {
+    "node": ">= 18"
+  },
   "prettier": "@snapshot-labs/prettier-config",
   "dependencies": {
+    "@apollo/client": "^3.7.12",
+    "@aws-sdk/client-s3": "^3.316.0",
+    "compression": "^1.7.4",
     "cors": "^2.8.5",
-    "express": "^4.18.2"
+    "express": "^4.18.2",
+    "graphql": "^16.6.0",
+    "winston": "^3.8.2"
   },
   "devDependencies": {
-    "@snapshot-labs/eslint-config": "^0.1.0-beta.0",
+    "@snapshot-labs/eslint-config": "^0.1.0-beta.9",
     "@snapshot-labs/prettier-config": "^0.1.0-beta.7",
+    "@types/compression": "^1.7.2",
     "@types/cors": "^2.8.13",
     "@types/express": "^4.17.17",
+    "@types/jest": "^29.5.1",
     "@types/node": "^18.15.11",
     "@typescript-eslint/eslint-plugin": "^5.59.0",
     "dotenv": "^16.0.3",
     "eslint": "^8.38.0",
     "eslint-plugin-prettier": "^4.2.1",
+    "jest": "^29.5.0",
+    "jest-environment-node-single-context": "^29.0.0",
     "nodemon": "^2.0.22",
+    "prettier": "^2.8.8",
+    "ts-jest": "^29.1.0",
     "ts-node": "^10.9.1",
     "typescript": "^5.0.4"
   }

src/api.ts

@@ -0,0 +1,77 @@
+import express from 'express';
+import { rpcError, rpcSuccess, storageEngine } from './helpers/utils';
+import log from './helpers/log';
+import { queues } from './lib/queue';
+import { name, version } from '../package.json';
+import VotesReport from './lib/votesReport';
+
+const router = express.Router();
+
+router.get('/', (req, res) => {
+  const commit = process.env.COMMIT_HASH || '';
+  const v = commit ? `${version}#${commit.substr(0, 7)}` : version;
+  return res.json({
+    name,
+    version: v
+  });
+});
+
+router.post('/votes/generate', async (req, res) => {
+  log.info(`[http] POST /votes/generate`);
+
+  const body = req.body || {};
+  const event = body.event.toString();
+  const id = body.id.toString().replace('proposal/', '');
+
+  if (req.headers['authenticate'] !== process.env.WEBHOOK_AUTH_TOKEN?.toString()) {
+    return rpcError(res, 'UNAUTHORIZED', id);
+  }
+
+  if (!event || !id) {
+    return rpcError(res, 'Invalid Request', id);
+  }
+
+  if (event !== 'proposal/end') {
+    return rpcSuccess(res, 'Event skipped', id);
+  }
+
+  try {
+    await new VotesReport(id, storageEngine(process.env.VOTE_REPORT_SUBDIR)).canBeCached();
+    queues.add(id);
+    return rpcSuccess(res, 'Cache file generation queued', id);
+  } catch (e) {
+    log.error(e);
+    return rpcError(res, 'INTERNAL_ERROR', id);
+  }
+});
+
+router.post('/votes/:id', async (req, res) => {
+  const { id } = req.params;
+  log.info(`[http] POST /votes/${id}`);
+
+  const votesReport = new VotesReport(id, storageEngine(process.env.VOTE_REPORT_SUBDIR));
+
+  try {
+    const file = await votesReport.cachedFile();
+
+    if (typeof file === 'string') {
+      res.header('Content-Type', 'text/csv');
+      res.attachment(votesReport.filename);
+      return res.send(Buffer.from(file));
+    }
+
+    try {
+      await votesReport.canBeCached();
+      queues.add(id);
+      return rpcError(res, 'PENDING_GENERATION', id);
+    } catch (e: any) {
+      log.error(e);
+      rpcError(res, e, id);
+    }
+  } catch (e) {
+    log.error(e);
+    return rpcError(res, 'INTERNAL_ERROR', id);
+  }
+});
+
+export default router;

src/helpers/log.ts

@@ -0,0 +1,9 @@
+import winston from 'winston';
+
+const log = winston.createLogger({
+  transports: new winston.transports.Console({
+    format: winston.format.combine(winston.format.colorize(), winston.format.simple())
+  })
+});
+
+export default log;