feat: add api endpoint to generate and serve cached file

.env.example

@@ -1 +1,6 @@
 HUB_URL=https://hub.snapshot.org
+AWS_ACCESS_KEY_ID=
+AWS_REGION=
+AWS_SECRET_ACCESS_KEY=
+AWS_BUCKET_NAME=
+WEBHOOK_AUTH_TOKEN=

.github/workflows/lint.yml

@@ -1,7 +1,7 @@
-name: Build and Run Tests
+name: Run Lint
 on: [push]
 jobs:
-  build-test:
+  build-lint:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3

.github/workflows/test.yml

@@ -0,0 +1,14 @@
+name: Build and Run Tests
+on: [push]
+jobs:
+  build-test:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+          cache: 'yarn'
+      - run: yarn install
+      - run: echo -en "HUB_URL=https://hub.snapshot.org" > .env
+      - run: yarn test

.gitignore

@@ -4,6 +4,7 @@ dist
 build
 .env
 coverage
+tmp
 
 # Remove some common IDE working directories
 .idea

LICENCE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) Snapshot Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,55 +1,132 @@
 # Snapshot/Sidekick
 
-Sidekick is the service serving all proposals' votes CSV report
+Sidekick is the service serving all proposal's votes CSV report
 
-<hr>
+-------------------
 
 This service is exposing an API endpoint expecting a closed proposal ID, and will
 return a CSV file with all the given proposal's votes.
 
-NOTE: CSV files are generated only once, then cached, making this service a cache middleware
-for snapshot-hub, for proposals' votes.
+> NOTE: CSV files are generated only once, then cached, making this service a cache middleware between snapshot-hub and UI
 
 ## Project Setup
 
+### Requirements
+
+node >= 18.0.0
+
 ### Dependencies
 
 Install the dependencies
 
-```
+```bash
 yarn
 ```
 
-_This project does not require a database, but may need larger server storage capacity for the cached files_
+_This project does not require a database, but requires a [storage engine](#storage-engine)_
 
 ### Configuration
 
-Edit the hub API url in the `.env` file if needed
+Copy `.env.example`, rename it to `.env` and edit the hub API url in the `.env` file if needed
 
-```
+```bash
 HUB_URL=https://hub.snapshot.org
 ```
 
-## Compiles and hot-reloads for development
+If you are using AWS as storage engine, set all the required `AWS_` config keys.
+
+### Storage engine
+
+This script is shipped with 2 storage engine:
+
+- `AWS`: All cached files will be stored on Amazon S3 storage
+- `File`: All cached files will be stored locally, in the `tmp` folder (used for dev environment and testing)
+
+You can toggle the cache engine in `helpers/utils.ts`, when importing the storage engine
 
+```typescript
+// For File (default) storage engine
+import StorageEngine from '../lib/storage/file';
+
+// For AWS storage engine
+import StorageEngine from '../lib/storage/aws';
 ```
+
+### Compiles and hot-reloads for development
+
+```bash
 yarn dev
 ```
 
-## Linting and typecheck
+## Linting, typecheck and test
 
-```
+```bash
 yarn lint
 yarn typecheck
+yarn test
 ```
 
-## Build for production
+## Usage
+
+Retrieving and generating the cache file have their own respective endpoint
+
+### Fetch a cache file
+
+Send a POST request with a proposal ID
 
+```bash
+curl -X POST localhost:3000/votes/[PROPOSAL-ID]
 ```
+
+When cached, this request will respond with a stream to a CSV file.
+
+On all other cases, it will respond with a [JSON-RPC 2.0](https://www.jsonrpc.org/specification) error response:
+
+```bash
+{
+  "jsonrpc":"2.0",
+  "error":{
+    "code": CODE,
+    "message": MESSAGE
+  },
+  "id": PROPOSAL-ID
+}
+```
+
+| Description                         | `CODE` | `MESSAGE`           |
+| ----------------------------------- | ------ | ------------------- |
+| When the proposal does not exist    | -40001 | PROPOSAL_NOT_FOUND  |
+| When the proposal is not closed     | -40004 | PROPOSAL_NOT_CLOSED |
+| When the file is pending generation | -40010 | PENDING_GENERATION  |
+| Other/Unknown/Server Error          | -32603 | INTERNAL_ERROR      |
+
+Furthermore, when votes report can be cached, but does not exist yet, a cache generation task will be queued. This enable cache to be generated on-demand.
+
+### Generate a cache file
+
+Send a POST request with a body following the [Webhook event object](https://docs.snapshot.org/tools/webhooks).
+
+```bash
+curl -X POST localhost:3000/votes/generate \
+-H "Authenticate: WEBHOOK_AUTH_TOKEN" \
+-H "Content-Type: application/json" \
+-d '{"id": "proposal/[PROPOSAL-ID]", "event": "proposal/end"}'
+```
+
+- On success, will respond with a success [JSON-RPC 2.0](https://www.jsonrpc.org/specification) message
+- On error, will respond with the same result and codes as the `fetch` endpoint above
+
+The endpoint has been designed to receive events from snapshot webhook service.
+
+Do not forget to set `WEBHOOK_AUTH_TOKEN` in the `.env` file
+
+## Build for production
+
+```bash
 yarn build
 yarn start
 ```
 
 ## License
 
-[MIT](https://github.com/snapshot-labs/envelop-ui/blob/bootstrap-app/LICENSE)
+[MIT](https://github.com/snapshot-labs/snapshot-sidekick/blob/main/LICENCE)

jest.config.ts

@@ -0,0 +1,20 @@
+/*
+ * For a detailed explanation regarding each configuration property and type check, visit:
+ * https://jestjs.io/docs/configuration
+ */
+
+export default {
+  clearMocks: true,
+  collectCoverage: true,
+  coverageDirectory: 'coverage',
+  coverageProvider: 'v8',
+
+  // An array of regexp pattern strings used to skip coverage collection
+  coveragePathIgnorePatterns: ['/node_modules/', '<rootDir>/dist/', '<rootDir>/test/fixtures/'],
+
+  preset: 'ts-jest',
+  testEnvironment: 'jest-environment-node-single-context',
+  setupFiles: ['dotenv/config'],
+  //setupFilesAfterEnv: ['<rootDir>src/setupTests.ts'],
+  moduleFileExtensions: ['js', 'ts']
+};

package.json

@@ -10,27 +10,45 @@
     "build": "tsc -p tsconfig.json",
     "dev": "nodemon src/index.ts -e js,ts",
     "start": "node dist/src/index.js",
-    "typecheck": "tsc"
+    "typecheck": "tsc --noEmit",
+    "test": "jest"
   },
   "eslintConfig": {
-    "extends": "@snapshot-labs"
+    "extends": "@snapshot-labs",
+    "rules": {
+      "no-throw-literal": "error"
+    }
+  },
+  "engines": {
+    "node": ">= 18"
   },
   "prettier": "@snapshot-labs/prettier-config",
   "dependencies": {
+    "@apollo/client": "^3.7.12",
+    "@aws-sdk/client-s3": "^3.316.0",
+    "compression": "^1.7.4",
     "cors": "^2.8.5",
-    "express": "^4.18.2"
+    "express": "^4.18.2",
+    "graphql": "^16.6.0",
+    "winston": "^3.8.2"
   },
   "devDependencies": {
-    "@snapshot-labs/eslint-config": "^0.1.0-beta.0",
+    "@snapshot-labs/eslint-config": "^0.1.0-beta.9",
     "@snapshot-labs/prettier-config": "^0.1.0-beta.7",
+    "@types/compression": "^1.7.2",
     "@types/cors": "^2.8.13",
     "@types/express": "^4.17.17",
+    "@types/jest": "^29.5.1",
     "@types/node": "^18.15.11",
     "@typescript-eslint/eslint-plugin": "^5.59.0",
     "dotenv": "^16.0.3",
     "eslint": "^8.38.0",
     "eslint-plugin-prettier": "^4.2.1",
+    "jest": "^29.5.0",
+    "jest-environment-node-single-context": "^29.0.0",
     "nodemon": "^2.0.22",
+    "prettier": "^2.8.8",
+    "ts-jest": "^29.1.0",
     "ts-node": "^10.9.1",
     "typescript": "^5.0.4"
   }

src/api.ts

@@ -0,0 +1,68 @@
+import express from 'express';
+import { voteReportWithStorage, rpcError, rpcSuccess } from './helpers/utils';
+import log from './helpers/log';
+import { queues } from './lib/queue';
+
+const router = express.Router();
+
+router.post('/votes/generate', async (req, res) => {
+  log.info(`[http] POST /votes/generate`);
+
+  const body = req.body || {};
+  const event = body.event.toString();
+  const id = body.id.toString().replace('proposal/', '');
+
+  if (req.headers['authenticate'] !== process.env.WEBHOOK_AUTH_TOKEN?.toString()) {
+    return rpcError(res, 'UNAUTHORIZED', id);
+  }
+
+  if (!event || !id) {
+    return rpcError(res, 'Invalid Request', id);
+  }
+
+  if (event !== 'proposal/end') {
+    return rpcSuccess(res, 'Event skipped', id);
+  }
+
+  try {
+    await voteReportWithStorage(id).canBeCached();
+    queues.add(id);
+    return rpcSuccess(res, 'Cache file generation queued', id);
+  } catch (e) {
+    log.error(e);
+    return rpcError(res, 'INTERNAL_ERROR', id);
+  }
+});
+
+router.post('/votes/:id', async (req, res) => {
+  const { id } = req.params;
+  log.info(`[http] POST /votes/${id}`);
+
+  const votesReport = voteReportWithStorage(id);
+
+  try {
+    const file = await votesReport.cachedFile();
+
+    if (typeof file === 'string') {
+      res.header('Content-Type', 'text/csv');
+      res.attachment(votesReport.filename);
+      return res.send(Buffer.from(file));
+    }
+
+    votesReport
+      .canBeCached()
+      .then(() => {
+        queues.add(id);
+        return rpcError(res, 'PENDING_GENERATION', id);
+      })
+      .catch((e: any) => {
+        log.error(e);
+        rpcError(res, e, id);
+      });
+  } catch (e) {
+    log.error(e);
+    return rpcError(res, 'INTERNAL_ERROR', id);
+  }
+});
+
+export default router;

src/helpers/log.ts

@@ -0,0 +1,9 @@
+import winston from 'winston';
+
+const log = winston.createLogger({
+  transports: new winston.transports.Console({
+    format: winston.format.combine(winston.format.colorize(), winston.format.simple())
+  })
+});
+
+export default log;