Skip to content

Commit

Permalink
Merge branch 'master' into more-strict-rate-limit-for-spammer
Browse files Browse the repository at this point in the history
  • Loading branch information
wa0x6e committed Dec 22, 2023
2 parents e670324 + c3e79e1 commit c37f0d2
Show file tree
Hide file tree
Showing 23 changed files with 871 additions and 343 deletions.
2 changes: 1 addition & 1 deletion jest.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ export default {
coverageProvider: 'v8',

// An array of regexp pattern strings used to skip coverage collection
coveragePathIgnorePatterns: ['/node_modules/', '<rootDir>/dist/', '<rootDir>/test/fixtures/'],
coveragePathIgnorePatterns: ['/node_modules/', '<rootDir>/build/', '<rootDir>/test/fixtures/'],

preset: 'ts-jest',
testEnvironment: 'jest-environment-node-single-context',
Expand Down
8 changes: 4 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@
"@ethersproject/wallet": "^5.6.2",
"@shutter-network/shutter-crypto": "^1.0.0",
"@snapshot-labs/pineapple": "^1.1.0",
"@snapshot-labs/snapshot-metrics": "^1.4.0",
"@snapshot-labs/snapshot-sentry": "^1.5.2",
"@snapshot-labs/snapshot.js": "^0.8.3",
"@snapshot-labs/snapshot-metrics": "^1.4.1",
"@snapshot-labs/snapshot-sentry": "^1.5.4",
"@snapshot-labs/snapshot.js": "^0.9.8",
"bluebird": "^3.7.2",
"connection-string": "^1.0.1",
"cors": "^2.8.5",
Expand All @@ -48,7 +48,7 @@
"winston": "^3.8.2"
},
"devDependencies": {
"@snapshot-labs/eslint-config": "^0.1.0-beta.13",
"@snapshot-labs/eslint-config": "^0.1.0-beta.15",
"@snapshot-labs/prettier-config": "^0.1.0-beta.7",
"@types/jest": "^29.5.2",
"@types/node": "^14.0.13",
Expand Down
65 changes: 65 additions & 0 deletions scripts/hibernate.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import 'dotenv/config';
import db from '../src/helpers/mysql';
import networks from '@snapshot-labs/snapshot.js/src/networks.json';

async function main() {
if (process.argv.length < 2) {
console.error(`Usage: yarn ts-node scripts/hibernate.ts run|preview`);
return process.exit(1);
}

const [, , action] = process.argv;

const commands = {
preview: `SELECT COUNT(id) as count from toHibernate`,
run: `UPDATE spaces SET hibernated = 1 where spaces.id IN (SELECT id FROM toHibernate)`
};

if (!commands[action]) {
console.error(`First argument should be either "run" or "preview"`);
return process.exit(1);
}

const liveNetworks = Object.values(networks)
.filter((network: any) => !network.testnet)
.map((network: any) => network.key);

const query = `
WITH toHibernate AS (
WITH data AS (
SELECT
id,
(SELECT MAX(created) FROM proposals WHERE space = spaces.id LIMIT 1) AS lastProposalEndDate
FROM spaces
WHERE hibernated = 0
)
SELECT
id, lastProposalEndDate
FROM data
WHERE
# Filtering out spaces that have not been active in the past year
FROM_UNIXTIME(lastProposalEndDate) < DATE_SUB(CURRENT_DATE, INTERVAL 1 YEAR)
)
${commands[action]};
`;

const results = await db.queryAsync(query, liveNetworks);

if (action === 'preview') {
console.log(`Spaces eligible for hibernation: ${results[0].count}`);
} else {
console.log(`${results.message}`);
}
}

(async () => {
try {
await main();
process.exit(0);
} catch (e) {
console.error(e);
process.exit(1);
}
})();
24 changes: 6 additions & 18 deletions src/api.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,16 @@
import express from 'express';
import snapshot from '@snapshot-labs/snapshot.js';
import relayer from './helpers/relayer';
import { addOrUpdateSpace } from './helpers/actions';
import { getSpaceENS } from './helpers/ens';
import { updateProposalAndVotes } from './scores';
import typedData from './ingestor';
import { sendError, verifyAuth } from './helpers/utils';
import { flagEntity } from './helpers/moderation';
import log from './helpers/log';
import { name, version } from '../package.json';
import { capture } from '@snapshot-labs/snapshot-sentry';
import poke from './helpers/poke';

const router = express.Router();
const network = process.env.NETWORK || 'testnet';
const SNAPSHOT_ENV = process.env.NETWORK || 'testnet';

const maintenanceMsg = 'update in progress, try later';

Expand All @@ -32,7 +30,7 @@ router.get('/', (req, res) => {
const v = commit ? `${version}#${commit.substr(0, 7)}` : version;
return res.json({
name,
network,
SNAPSHOT_ENV,
version: v,
relayer: relayer.address
});
Expand All @@ -51,20 +49,10 @@ router.get('/scores/:proposalId', async (req, res) => {
});

router.get('/spaces/:key/poke', async (req, res) => {
const { key } = req.params;
try {
let space = false;
const result = await getSpaceENS(key);
if (snapshot.utils.validateSchema(snapshot.schemas.space, result) === true) space = result;
if (space) {
await addOrUpdateSpace(key, space);
// spaces[key] = space;
}
return res.json(space);
} catch (e) {
capture(e);
log.warn(`[api] Load space failed ${key}`);
return res.json(false);
return res.json(await poke(req.params.key));
} catch (e: any) {
return res.json({ error: e });
}
});

Expand Down
7 changes: 4 additions & 3 deletions src/helpers/actions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ export async function addOrUpdateSpace(space: string, settings: any) {
if (!settings?.name) return false;
const ts = (Date.now() / 1e3).toFixed();
const query =
'INSERT IGNORE INTO spaces SET ? ON DUPLICATE KEY UPDATE updated = ?, settings = ?, name = ?';
'INSERT IGNORE INTO spaces SET ? ON DUPLICATE KEY UPDATE updated = ?, settings = ?, name = ?, hibernated = 0';
await db.queryAsync(query, [
{
id: space,
Expand Down Expand Up @@ -36,7 +36,7 @@ export async function getProposal(space, id) {
}

export async function getSpace(id: string, includeDeleted = false) {
const query = `SELECT settings, deleted, flagged, verified FROM spaces WHERE id = ? AND deleted in (?) LIMIT 1`;
const query = `SELECT settings, deleted, flagged, verified, hibernated FROM spaces WHERE id = ? AND deleted in (?) LIMIT 1`;
const spaces = await db.queryAsync(query, [id, includeDeleted ? [0, 1] : [0]]);

if (!spaces[0]) return false;
Expand All @@ -45,7 +45,8 @@ export async function getSpace(id: string, includeDeleted = false) {
...jsonParse(spaces[0].settings, {}),
deleted: spaces[0].deleted === 1,
verified: spaces[0].verified === 1,
flagged: spaces[0].flagged === 1
flagged: spaces[0].flagged === 1,
hibernated: spaces[0].hibernated === 1
};
}

Expand Down
20 changes: 0 additions & 20 deletions src/helpers/ens.ts

This file was deleted.

13 changes: 10 additions & 3 deletions src/helpers/moderation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ export async function loadModerationData(url = moderationURL): Promise<boolean>
}

flaggedIps = body.flaggedIps;
flaggedAddresses = body.flaggedAddresses;
flaggedAddresses = body.flaggedAddresses || [];
flaggedAddresses = flaggedAddresses.map((address: string) => address.toLowerCase());
flaggedProposalTitleKeywords = body.flaggedProposalTitleKeywords;
flaggedProposalBodyKeywords = body.flaggedProposalBodyKeywords;

Expand All @@ -45,7 +46,7 @@ export function flagEntity({ type, action, value }) {
if (!['proposal', 'space'].includes(type)) throw new Error('invalid type');
if (type === 'proposal' && !['flag', 'unflag'].includes(action))
throw new Error('invalid action');
if (type === 'space' && !['flag', 'unflag', 'verify'].includes(action))
if (type === 'space' && !['flag', 'unflag', 'verify', 'hibernate', 'reactivate'].includes(action))
throw new Error('invalid action');

let query;
Expand All @@ -62,9 +63,15 @@ export function flagEntity({ type, action, value }) {
case 'proposal-flag':
query = `UPDATE proposals SET flagged = 1 WHERE id = ? LIMIT 1`;
break;
case 'proposal-flag':
case 'proposal-unflag':
query = `UPDATE proposals SET flagged = 0 WHERE id = ? LIMIT 1`;
break;
case 'space-hibernate':
query = `UPDATE spaces SET hibernated = 1 WHERE id = ? LIMIT 1`;
break;
case 'space-reactivate':
query = `UPDATE spaces SET hibernated = 0 WHERE id = ? LIMIT 1`;
break;
}

if (!query) throw new Error('invalid query');
Expand Down
47 changes: 47 additions & 0 deletions src/helpers/poke.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import snapshot from '@snapshot-labs/snapshot.js';
import { capture } from '@snapshot-labs/snapshot-sentry';
import { addOrUpdateSpace } from './actions';

type Space = Record<string, any>;

const DEFAULT_NETWORK = process.env.DEFAULT_NETWORK ?? '1';
const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org';

export default async function poke(id: string): Promise<Space> {
const space = await getSpaceENS(id);

try {
if (snapshot.utils.validateSchema(snapshot.schemas.space, space) !== true) {
return Promise.reject('invalid space format');
}

await addOrUpdateSpace(id, space);

return space;
} catch (e: any) {
capture(e);
return Promise.reject('unable to save the space');
}
}

async function getSpaceENS(id: string): Promise<Space> {
const uri = await snapshot.utils.getSpaceUri(id, DEFAULT_NETWORK, { broviderUrl });

if (uri) {
if (!isValidUri(uri)) {
return Promise.reject('TXT record is not a valid uri');
}

try {
return await snapshot.utils.getJSON(uri);
} catch (e) {
return Promise.reject(`${uri} is not a valid JSON file`);
}
}

return Promise.reject(`missing snapshot TXT record on ENS name ${id}`);
}

function isValidUri(uri: string): boolean {
return /^(ip[fn]s|https?):\/\//.test(uri);
}
1 change: 0 additions & 1 deletion src/helpers/shutter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ export async function shutterDecrypt(encryptedMsg: string, decryptionKey: string
try {
const decryptedMsg = await decrypt(arrayify(encryptedMsg), arrayify(decryptionKey));
const result = toUtf8String(decryptedMsg);
log.info(`[shutter] decrypted ${JSON.stringify(result)}`);
return result;
} catch (e) {
capture(e);
Expand Down
1 change: 1 addition & 0 deletions src/helpers/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ export function hasStrategyOverride(strategies: any[]) {
'"cyberkongz"',
'"cyberkongz-v2"',
'"delegation"',
'"delegation-with-cap"',
'"delegation-with-overrides"',
'"erc20-balance-of-delegation"',
'"erc20-balance-of-fixed-total"',
Expand Down
40 changes: 21 additions & 19 deletions src/writer/proposal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ import log from '../helpers/log';
import { ACTIVE_PROPOSAL_BY_AUTHOR_LIMIT, getSpaceLimits } from '../helpers/limits';
import { capture } from '@snapshot-labs/snapshot-sentry';
import { flaggedAddresses } from '../helpers/moderation';
import { validateSpaceSettings } from './settings';

const network = process.env.NETWORK || 'testnet';
const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org';
const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org';

Expand All @@ -36,6 +36,22 @@ export const getProposalsCount = async (space, author) => {
return await db.queryAsync(query, [space, author]);
};

async function validateSpace(space: any) {
if (!space) {
return Promise.reject('unknown space');
}

if (space.hibernated) {
return Promise.reject('space hibernated');
}

try {
await validateSpaceSettings(space);
} catch (e) {
return Promise.reject(e);
}
}

export async function verify(body): Promise<any> {
const msg = jsonParse(body.msg);
const created = parseInt(msg.timestamp);
Expand All @@ -60,27 +76,13 @@ export async function verify(body): Promise<any> {

const space = await getSpace(msg.space);

if (!space) {
return Promise.reject('unknown space');
try {
await validateSpace(space);
} catch (e) {
return Promise.reject(`invalid space settings: ${e}`);
}

space.id = msg.space;
const hasTicket = space.strategies.some(strategy => strategy.name === 'ticket');
const hasVotingValidation =
space.voteValidation?.name && !['any'].includes(space.voteValidation.name);

if (hasTicket && !hasVotingValidation && network !== 'testnet') {
return Promise.reject('space with ticket requires voting validation');
}

const hasProposalValidation =
(space.validation?.name && space.validation.name !== 'any') ||
space.filters?.minScore ||
space.filters?.onlyMembers;

if (!hasProposalValidation && network !== 'testnet') {
return Promise.reject('space missing proposal validation');
}

// if (msg.payload.start < created) return Promise.reject('invalid start date');

Expand Down
Loading

0 comments on commit c37f0d2

Please sign in to comment.