Even more improved token validation script (#1153)

* typescript sourcemap: true

* implemented canOnlyAddOneToken(), detectDuplicateSymbol()

* ValidationError.INVALID_MINT: some mints are PDAs and won't be on the ed25519 curve. just check for base58 decodability

* Record.Community Validated: bool -> string

* rm parse.ts

* validate.ts: complete validation functions

* exec() was showing the output of 'git show' even when stdout was redirected to the function. This is needed to make it not output anything to the console no matter what

* main.ts: functionality complete

* removed unused function in validate.ts

* exceptions: some tokens don't need to have communityValidated: true

* use ValidatedTokensData which existed already, change 'Community Validated' to a bool

* remove exceptions for community validated

* validMintAddress(): mints don't have to be on the edd25519 curve, remove entirely

* ValidatedTokensData: keep track of which line it was in the CSV

* rework duplicate symbol detection, abstract out file reading, csv parsing and struct hydrating

* noEditsToPreviousLinesAllowed()

* better validation error messages.  a better message tells you what to do

* rework detectDuplicateSymbol()

* split logic out, so one can run it with Github Actions (which doesn't accept CLI arguments) or as a CLI program (which accepts arguments) with node dist/cli.js <filename>

* revive github action for validate-PR

* improve duplicate symbol detection: people may not submit tokens that are duplicates of already existing duplicate symbols

* exit code is the number of errors. this is useful for github actions reporting

.github/workflows/validate-PR.yml

@@ -20,10 +20,8 @@ jobs:
 
       ## for local act testing
       # - run: npm install -g yarn
-      
+
       - run: yarn install
 
       - name: Validate PR
-        run: yarn validate-PR
-
-
+        run: yarn validate-PR >> $GITHUB_STEP_SUMMARY

package.json

@@ -21,7 +21,9 @@
     "@actions/exec": "^1.1.1",
     "@actions/github": "^5.1.1",
     "@solana/web3.js": "^1.73.2",
+    "@types/minimist": "^1.2.5",
     "csv-writer": "^1.6.0",
+    "minimist": "^1.2.8",
     "node-downloader-helper": "^2.1.6",
     "node-fetch": "^2.6.6"
   },

src/cli.ts

@@ -0,0 +1,13 @@
+import { validateValidatedTokensCsv } from "./logic";
+import minimist from "minimist";
+// CLI entrypoint which accepts an argument
+(async () => {
+    try {
+        const argv = minimist(process.argv.slice(2));
+        const returnCode = await validateValidatedTokensCsv(argv._[0]);
+        process.exit(returnCode);
+    }
+    catch (error: any) {
+        console.log(error.message)
+    }
+})();

src/logic.ts

@@ -0,0 +1,91 @@
+import * as core from "@actions/core";
+import { exec } from "@actions/exec";
+import { canOnlyAddOneToken, detectDuplicateSymbol, detectDuplicateMints as detectDuplicateMints, validMintAddress, noEditsToPreviousLinesAllowed } from "./utils/validate";
+import { ValidatedTokensData } from "./types/types";
+import { indexToLineNumber } from "./utils/validate";
+import { parse } from "csv-parse/sync";
+import fs from "fs";
+
+export async function validateValidatedTokensCsv(filename: string): Promise<number> {
+    const [records, recordsRaw] = parseCsv(filename);
+
+    const recordsPreviousRaw = await gitPreviousVersion("validated-tokens.csv");
+    fs.writeFileSync(".validated-tokens-0.csv", recordsPreviousRaw);
+    const [recordsPrevious, _] = parseCsv(".validated-tokens-0.csv")
+
+    let duplicateSymbols;
+    let duplicateMints;
+    let attemptsToAddMultipleTokens;
+    let invalidMintAddresses;
+    let notCommunityValidated;
+    let noEditsAllowed;
+
+    duplicateSymbols = detectDuplicateSymbol(recordsPrevious, records);
+    duplicateMints = detectDuplicateMints(records);
+    attemptsToAddMultipleTokens = canOnlyAddOneToken(recordsPrevious, records)
+    invalidMintAddresses = validMintAddress(records);
+    noEditsAllowed = noEditsToPreviousLinesAllowed(recordsPrevious, records);
+    // notCommunityValidated = validCommunityValidated(records);
+
+    console.log("No More Duplicate Symbols:", duplicateSymbols);
+    console.log("Duplicate Mints:", duplicateMints);
+    console.log("Attempts to Add Multiple Tokens:", attemptsToAddMultipleTokens);
+    console.log("Invalid Mint Addresses:", invalidMintAddresses);
+    console.log("Not Community Validated:", notCommunityValidated);
+    console.log("Edits to Existing Tokens:", noEditsAllowed);
+    return (duplicateSymbols + duplicateMints + attemptsToAddMultipleTokens + invalidMintAddresses + noEditsAllowed)
+}
+
+// Get previous version of validated-tokens.csv from last commit
+async function gitPreviousVersion(path: string): Promise<any> {
+    let prevVersion = "";
+    let gitCmdError = "";
+
+    try {
+        await exec("git", ["show", `origin/main:${path}`], {
+            listeners: {
+                stdout: (data: Buffer) => {
+                    prevVersion += data.toString();
+                },
+                stderr: (data: Buffer) => {
+                    gitCmdError += data.toString();
+                },
+            },
+            silent: true
+        });
+    } catch (error: any) {
+        core.setFailed(error.message);
+    }
+
+    if (gitCmdError) {
+        core.setFailed(gitCmdError);
+    }
+    return prevVersion;
+}
+
+function parseCsv(filename: string): [ValidatedTokensData[], string] {
+    const recordsRaw = fs.readFileSync(filename, "utf8")
+    const r = parse(recordsRaw, {
+        columns: true,
+        skip_empty_lines: true,
+    });
+    const records = csvToRecords(r);
+    return [records, recordsRaw];
+}
+
+function csvToRecords(r: any): ValidatedTokensData[] {
+    const records: ValidatedTokensData[] = [];
+    r.forEach((record: any, i: number) => {
+        const rec: ValidatedTokensData = {
+            Name: record.Name,
+            Symbol: record.Symbol,
+            Mint: record.Mint,
+            Decimals: record.Decimals,
+            LogoURI: record.LogoURI,
+            "Community Validated": JSON.parse(record["Community Validated"]),
+            Line: indexToLineNumber(i)
+        };
+        records.push(rec);
+    });
+    return records;
+}

src/main.ts

@@ -1,75 +1,13 @@
 import * as core from "@actions/core";
-import { exec } from "@actions/exec";
-import { parseGitPatch } from "./utils/parse";
-import { validateGitPatch } from "./utils/validate";
-import { getValidated } from "./utils/get-jup-strict";
-import { ValidatedSet, ValidationError } from "./types/types";
-import { parse } from "csv-parse/sync";
-import fs from "fs";
-import assert from "assert";
-
-function validateValidatedTokensCsv() {
-  const records = parse(fs.readFileSync("validated-tokens.csv", "utf8"), {
-    columns: true,
-    skip_empty_lines: true,
-  });
-  assert.deepStrictEqual(Object.keys(records[0]), [
-    "Name",
-    "Symbol",
-    "Mint",
-    "Decimals",
-    "LogoURI",
-    "Community Validated",
-  ]);
-}
-
-// Validates diff between validated-tokens.csv in the branch vs origin/main
-async function getDiffAndValidate(): Promise<void> {
-  let gitDiff = "";
-  let gitDiffError = "";
-
+import { validateValidatedTokensCsv } from "./logic";
+// Github Actions entrypoint
+(async () => {
   try {
-    await exec("git", ["diff", "origin/main", "validated-tokens.csv"], {
-      listeners: {
-        stdout: (data: Buffer) => {
-          gitDiff += data.toString();
-        },
-        stderr: (data: Buffer) => {
-          gitDiffError += data.toString();
-        },
-      },
-    });
-  } catch (error: any) {
-    core.setFailed(error.message);
-  }
-
-  if (gitDiffError) {
-    core.setFailed(gitDiffError);
+    const returnCode = await validateValidatedTokensCsv("validated-tokens.csv");
+    process.exit(returnCode);
   }
-
-  // core.debug(`Git diff: ${gitDiff}`)
-
-  // Get Jup tokens that are in the strict list to check for duplicates.
-  let validatedSet: ValidatedSet;
-  try {
-    validatedSet = await getValidated();
-
-    const errors: ValidationError[][] = [];
-
-    parseGitPatch(gitDiff).forEach((patch) => {
-      const patchErrors = validateGitPatch(patch, validatedSet);
-      if (patchErrors && patchErrors.length > 0) {
-        errors.push(patchErrors);
-      }
-    });
-
-    if (errors.length > 0) {
-      core.setFailed(errors.join(","));
-    }
-  } catch (error: any) {
+  catch (error: any) {
     core.setFailed(error.message);
+    console.log(error.message)
   }
-}
-
-validateValidatedTokensCsv();
-// getDiffAndValidate();
+})();

src/types/types.ts

@@ -68,12 +68,13 @@ export enum ValidationError {
   UNRELATED_CODE = "Changes to unrelated code are not allowed",
   MULTIPLE_TOKENS = "Only one token can be added at a time",
   DUPLICATE_NAME = "Token name already exists",
-  DUPLICATE_SYMBOL = "Token symbol already exists",
+  DUPLICATE_SYMBOL = "Token symbol already exists, please forbid even more duplicates",
   DUPLICATE_MINT = "Mint already exists",
-  INVALID_MINT = "Invalid mint address, not on ed25519 curve",
+  INVALID_MINT = "Invalid mint address, not base58 decodable",
   INVALID_DECIMALS = "Invalid decimals",
   INVALID_IMAGE_URL = "Invalid image URL",
   INVALID_COMMUNITY_VALIDATED = "Invalid community validated",
+  CHANGES_DISCOURAGED = "Tokens already in the CSV should not be edited"
 }
 
 export interface WormholeData {
@@ -97,5 +98,12 @@ export interface ValidatedTokensData {
   Mint: string;
   Decimals: string;
   LogoURI: string;
-  "Community Validated": "false" | "true";
+  "Community Validated": boolean;
+  Line: number;
 }
+
+export interface DuplicateSymbol {
+  Name: string;
+  Symbol: string;
+  Mint: string;
+}

src/utils/duplicate-symbols.ts

@@ -0,0 +1,94 @@
+import { DuplicateSymbol } from "../types/types";
+
+export const allowedDuplicateSymbols: DuplicateSymbol[] = [
+    {
+        Name: 'Sallar',
+        Symbol: 'ALL',
+        Mint: '5EjMX8pZkJtkbJwT5vzJhzTexBPhECFUrq5ndD3UkQD1',
+      },
+      {
+        Name: 'Arbitrum (Portal from Arbitrum)',
+        Symbol: 'ARB',
+        Mint: '8LH3QMo7xkMJx85Kg4pfiQY1g1ZgiVEe1KktSpaT89mP',
+      },
+      {
+        Name: 'AVAX (Portal)',
+        Symbol: 'AVAX',
+        Mint: 'KgV1GvrHQmRBY8sHQQeUKwTm2r2h8t4C8qt12Cw1HVE',
+      },
+      {
+        Name: 'Boo',
+        Symbol: 'BOO',
+        Mint: 'FfpyoV365c7iR8QQg5NHGCXQfahbqzY67B3wpzXkiLXr',
+      },
+      {
+        Name: 'Food',
+        Symbol: 'FOOD',
+        Mint: 'foodQJAztMzX1DKpLaiounNe2BDMds5RNuPC6jsNrDG',
+      },
+      {
+        Name: 'Helios Rising: Fuel',
+        Symbol: 'FUEL',
+        Mint: 'ZViNy4z9dquon7AVgr6neK1RCohTRFH8WTUMUsjhWhe',
+      },
+      {
+        Name: 'Starbots GEAR',
+        Symbol: 'GEAR',
+        Mint: '23WuycvPjEuzJTsBPBZqnbFZFcBtBKAMTowUDHwagkuD',
+      },
+      {
+        Name: 'GM',
+        Symbol: 'GM',
+        Mint: '3acxNNmfdKKZj9i35P4VDBFm74Ufdt8ojKWceVGynwC5',
+      },
+      {
+        Name: 'LILY',
+        Symbol: 'LILY',
+        Mint: '7FYvphuZtRxB7BZd8PZ65yZmEEuWYYdxiHubthyd38BE',
+      },
+      {
+        Name: 'MILK',
+        Symbol: 'MILK',
+        Mint: 'MLKmUCaj1dpBY881aFsrBwR9RUMoKic8SWT3u1q5Nkj',
+      },
+      {
+        Name: 'NANA Token',
+        Symbol: 'NANA',
+        Mint: 'HxRELUQfvvjToVbacjr9YECdfQMUqGgPYB68jVDYxkbr',
+      },
+      {
+        Name: 'NINJA TURTLES',
+        Symbol: 'NINJA',
+        Mint: 'DFrJxDoLMYt6bNYeNe8Wrjzj2UPUSLZLEMMYBLuTKcTk',
+      },
+      {
+        Name: 'Only Possible On Solana',
+        Symbol: 'OPOS',
+        Mint: 'BqVHWpwUDgMik5gbTciFfozadpE2oZth5bxCDrgbDt52',
+      },
+      {
+        Name: 'PEPESOLANA',
+        Symbol: 'PEPE',
+        Mint: 'CYuXNHURE8cF3rQc1687DZnmsWx9ATNG3mZPFmryu22S',
+      },
+      {
+        Name: 'Rocky',
+        Symbol: 'ROCKY',
+        Mint: '4icEZCrEYNop2ZaMMCkRHaNzkt6xG9BpijMCQV7mpw6Z',
+      },
+      {
+        Name: 'Soul Scanner',
+        Symbol: 'SOUL',
+        Mint: 'J4ywFdm8H7hjwKzCaEQujhkDRfCnRviVnHMvFNDAoLNQ',
+      },
+      {
+        Name: 'sRLY (Rally Solana)',
+        Symbol: 'sRLY',
+        Mint: 'sRLY3migNrkC1HLgqotpvi66qGkdNedqPZ9TJpAQhyh',
+      },
+      {
+        Name: 'WHEY',
+        Symbol: 'WHEY',
+        Mint: 'Ue4yjkPjA4QGis37eWbBsnqfzyK83BtY4AioDETp3Ab',
+      }
+]