Merge pull request #160 from subquery/add/proto-ts-codegen

Support for cosmos codegen

.eslintignore

@@ -6,3 +6,4 @@
 packages/**/dist/**
 packages/**/lib/**
 .eslintrc.js
+*.proto

.prettierignore

@@ -9,3 +9,4 @@
 *.tgz
 *.cmd
 *.sh
+*.proto

packages/common-cosmos/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Support protobuf to ts cosmos `codegen` (#160)
 
 ## [2.3.1] - 2023-07-31
 ### Changed

packages/common-cosmos/package.json

@@ -13,17 +13,29 @@
   "main": "dist/index.js",
   "license": "GPL-3.0",
   "dependencies": {
+    "@cosmology/telescope": "^0.104.0",
+    "@protobufs/cosmos": "^0.1.0",
+    "@protobufs/cosmos_proto": "^0.0.10",
+    "@protobufs/cosmwasm": "^0.1.1",
+    "@protobufs/gogoproto": "^0.0.10",
+    "@protobufs/google": "^0.0.10",
+    "@protobufs/ibc": "^0.1.0",
+    "@protobufs/tendermint": "^0.0.10",
     "@subql/common": "^2.4.0",
     "@subql/types-cosmos": "workspace:*",
+    "fs-extra": "^11.1.1",
     "js-yaml": "^4.1.0",
     "reflect-metadata": "^0.1.13"
   },
   "peerDependencies": {
     "class-transformer": "*",
-    "class-validator": "*"
+    "class-validator": "*",
+    "ejs": "*"
   },
   "devDependencies": {
     "@types/bn.js": "4.11.6",
+    "@types/ejs": "^3.1.2",
+    "@types/fs-extra": "^11",
     "@types/js-yaml": "^4.0.4",
     "@types/pino": "^6.3.12"
   }

packages/common-cosmos/src/codegen/codegen-controller.spec.ts

@@ -0,0 +1,71 @@
+// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+
+import path from 'path';
+import {loadFromJsonOrYaml} from '@subql/common';
+import {isProtoPath, prepareProtobufRenderProps, processProtoFilePath} from './codegen-controller';
+import {validateCosmosManifest} from './util';
+
+const PROJECT_PATH = path.join(__dirname, '../../test/protoTest1');
+
+describe('Codegen cosmos, protobuf to ts', () => {
+  it('process protobuf file paths', () => {
+    const p = './proto/cosmos/osmosis/poolmanager/v1beta1/swap_route.proto';
+    expect(processProtoFilePath(p)).toBe('./proto-interfaces/cosmos/osmosis/poolmanager/v1beta1/swap_route');
+  });
+  it('should output correct protobuf render props', () => {
+    const mockChainTypes = [
+      {
+        'osmosis.gamm.v1beta1': {
+          file: './proto/osmosis/gamm/v1beta1/tx.proto',
+          messages: ['MsgSwapExactAmountIn'],
+        },
+      },
+      {
+        'osmosis.poolmanager.v1beta1': {
+          file: './proto/osmosis/poolmanager/v1beta1/swap_route.proto',
+          messages: ['SwapAmountInRoute'],
+        },
+      },
+    ];
+    expect(prepareProtobufRenderProps(mockChainTypes, PROJECT_PATH)).toStrictEqual([
+      {
+        messageNames: ['MsgSwapExactAmountIn'],
+        path: './proto-interfaces/osmosis/gamm/v1beta1/tx',
+      },
+      {
+        messageNames: ['SwapAmountInRoute'],
+        path: './proto-interfaces/osmosis/poolmanager/v1beta1/swap_route',
+      },
+    ]);
+  });
+  it('Should throw if path to protobuf does not exist', () => {
+    const mockChainTypes = [
+      {
+        'osmosis.gamm.v1beta1': {
+          file: './protato/osmosis/gamm/v1beta1/tx.proto',
+          messages: ['MsgSwapExactAmountIn'],
+        },
+      },
+    ];
+    expect(() => prepareProtobufRenderProps(mockChainTypes, PROJECT_PATH)).toThrow(
+      'Error: chainType osmosis.gamm.v1beta1, file ./protato/osmosis/gamm/v1beta1/tx.proto does not exist'
+    );
+  });
+  it('ensure correct regex for protoPath', () => {
+    let p = './proto/cosmos/osmosis/gamm/v1beta1/tx.proto';
+    expect(isProtoPath(p, PROJECT_PATH)).toBe(true);
+    p = 'proto/cosmos/osmosis/gamm/v1beta1/tx.proto';
+    expect(isProtoPath(p, PROJECT_PATH)).toBe(true);
+    p = '../proto/cosmos/osmosis/gamm/v1beta1/tx.proto';
+    expect(isProtoPath(p, PROJECT_PATH)).toBe(false);
+    p = './protos/cosmos/osmosis/gamm/v1beta1/tx.proto';
+    expect(isProtoPath(p, PROJECT_PATH)).toBe(false);
+  });
+  it('Ensure correctness on Cosmos Manifest validate', () => {
+    const cosmosManifest = loadFromJsonOrYaml(path.join(PROJECT_PATH, 'project.yaml')) as any;
+    const ethManifest = loadFromJsonOrYaml(path.join(PROJECT_PATH, 'bad-cosmos-project.yaml')) as any;
+    expect(validateCosmosManifest(cosmosManifest)).toBe(true);
+    expect(validateCosmosManifest(ethManifest)).toBe(false);
+  });
+});

packages/common-cosmos/src/codegen/codegen-controller.ts

@@ -0,0 +1,118 @@
+// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+
+import fs from 'fs';
+import path from 'path';
+import telescope from '@cosmology/telescope';
+import {makeTempDir} from '@subql/common';
+import {CustomModule} from '@subql/types-cosmos';
+import {Data} from 'ejs';
+import {copySync} from 'fs-extra';
+import {TELESCOPE_OPTS} from './constants';
+
+const PROTO_INTERFACES_ROOT_DIR = 'src/types/proto-interfaces';
+const PROTO_INTERFACE_TEMPLATE_PATH = path.resolve(__dirname, './templates/proto-interface.ts.ejs');
+const TYPE_ROOT_DIR = 'src/types';
+
+interface ProtobufRenderProps {
+  messageNames: string[]; // all messages
+  path: string; // should process the file Path and concat with PROTO dir
+}
+type CosmosChainTypeDataType = Map<string, CustomModule> | Record<string, CustomModule>;
+
+export function processProtoFilePath(path: string): string {
+  // removes `./proto` and `.proto` suffix, converts all `.` to `/`
+  // should be able to accept more paths, not just from `proto directory`
+  return `./proto-interfaces/${path.replace(/^\.\/proto\/|\.proto$/g, '').replace(/\./g, '/')}`;
+}
+
+export function isProtoPath(filePath: string, projectPath: string): boolean {
+  // check if the protobuf files are under ./proto directory
+  return !!path.join(projectPath, filePath).startsWith(path.join(projectPath, './proto/'));
+}
+
+export function prepareProtobufRenderProps(
+  chainTypes: CosmosChainTypeDataType[],
+  projectPath: string
+): ProtobufRenderProps[] {
+  return chainTypes.flatMap((chainType) => {
+    return Object.entries(chainType).map(([key, value]) => {
+      const filePath = path.join(projectPath, value.file);
+      if (!fs.existsSync(filePath)) {
+        throw new Error(`Error: chainType ${key}, file ${value.file} does not exist`);
+      }
+      if (!isProtoPath(value.file, projectPath)) {
+        console.error(
+          `Codegen will not apply for this file: ${value.file} Please ensure it is under the ./proto directory if you want to run codegen on it`
+        );
+      }
+      return {
+        messageNames: value.messages,
+        path: processProtoFilePath(value.file),
+      };
+    });
+  });
+}
+
+export async function tempProtoDir(projectPath: string): Promise<string> {
+  const tmpDir = await makeTempDir();
+  const userProto = path.join(projectPath, './proto');
+  const commonProtoPaths = [
+    require('@protobufs/amino'),
+    require('@protobufs/confio'),
+    require('@protobufs/cosmos'),
+    require('@protobufs/cosmos_proto'),
+    require('@protobufs/gogoproto'),
+    require('@protobufs/google'),
+    require('@protobufs/ibc'),
+    require('@protobufs/tendermint'),
+  ];
+
+  commonProtoPaths.forEach((p) => {
+    // ensure output format is a dir
+    copySync(p, path.join(tmpDir, `${p.replace(path.dirname(p), '')}`));
+  });
+  copySync(userProto, tmpDir, {overwrite: true});
+  return tmpDir;
+}
+
+export async function generateProto(
+  chainTypes: CosmosChainTypeDataType[],
+  projectPath: string,
+  prepareDirPath: (path: string, recreate: boolean) => Promise<void>,
+  renderTemplate: (templatePath: string, outputPath: string, templateData: Data) => Promise<void>,
+  upperFirst: (string?: string) => string,
+  mkdirProto: (projectPath: string) => Promise<string>
+): Promise<void> {
+  let tmpPath: string;
+  try {
+    tmpPath = await mkdirProto(projectPath);
+    const protobufRenderProps = prepareProtobufRenderProps(chainTypes, projectPath);
+    const outputPath = path.join(projectPath, PROTO_INTERFACES_ROOT_DIR);
+    await prepareDirPath(path.join(projectPath, PROTO_INTERFACES_ROOT_DIR), true);
+
+    await telescope({
+      protoDirs: [tmpPath],
+      outPath: outputPath,
+      options: TELESCOPE_OPTS,
+    });
+    console.log('* Protobuf types generated !');
+
+    await renderTemplate(
+      PROTO_INTERFACE_TEMPLATE_PATH,
+      path.join(projectPath, TYPE_ROOT_DIR, 'CosmosMessageTypes.ts'),
+      {
+        props: {proto: protobufRenderProps},
+        helper: {upperFirst},
+      }
+    );
+    console.log('* Cosmos message wrappers generated !');
+  } catch (e: any) {
+    const errorMessage = e.message.startsWith('Dependency')
+      ? `Please add the missing protobuf file to ./proto directory`
+      : '';
+    throw new Error(`Failed to generate from protobufs. ${e.message}, ${errorMessage}`);
+  } finally {
+    fs.rmSync(tmpPath, {recursive: true, force: true});
+  }
+}

packages/common-cosmos/src/codegen/constants.ts

@@ -0,0 +1,68 @@
+// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+
+import {TelescopeOptions} from '@cosmology/types/types/telescope';
+
+export const TELESCOPE_OPTS: TelescopeOptions = {
+  removeUnusedImports: true,
+  tsDisable: {
+    patterns: ['**/*amino.ts', '**/*registry.ts'],
+  },
+  // experimentalGlobalProtoNamespace: true, //  [ 'v1beta1' ] concentratedliquidity
+  interfaces: {
+    enabled: true,
+    useUnionTypes: false,
+  },
+  prototypes: {
+    enabled: false,
+    addTypeUrlToDecoders: true,
+    addTypeUrlToObjects: true,
+    excluded: {
+      packages: [
+        'amino',
+        'gogoproto',
+        // 'google.api',
+        // 'ibc.core.port.v1',
+        // 'ibc.core.types.v1',
+      ],
+    },
+    methods: {
+      fromJSON: false,
+      toJSON: false,
+
+      encode: false,
+      decode: false,
+      fromPartial: false,
+
+      toSDK: false,
+      fromSDK: false,
+
+      toAmino: false,
+      fromAmino: false,
+      fromProto: false,
+      toProto: false,
+    },
+    parser: {
+      keepCase: false,
+    },
+    typingsFormat: {
+      duration: 'duration',
+      timestamp: 'date',
+      useExact: false,
+      useDeepPartial: false,
+    },
+  },
+  aminoEncoding: {
+    enabled: false,
+    exceptions: {},
+    useRecursiveV2encoding: true,
+  },
+  lcdClients: {
+    enabled: false,
+  },
+  rpcClients: {
+    // unsure if needed
+    enabled: false,
+    camelCase: true,
+  },
+};

packages/common-cosmos/src/codegen/index.ts

@@ -0,0 +1,5 @@
+// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+
+export * from './codegen-controller';
+export * from './util';

packages/common-cosmos/src/codegen/templates/proto-interface.ts.ejs

@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: Apache-2.0
+
+// Auto-generated , DO NOT EDIT
+import {CosmosMessage} from "@subql/types-cosmos";
+<% props.proto.forEach(function(proto) { %>
+import {<% proto.messageNames.forEach(function(msg, index) { %><%= helper.upperFirst(msg) %><% if (index < proto.messageNames.length - 1) { %>,<% } %><% }); %>} from "<%= proto.path %>";
+<% }); %>
+<% props.proto.forEach(function(proto) { %><% proto.messageNames.forEach(function(msg) { %>
+export type <%= helper.upperFirst(msg) %>Message = CosmosMessage<<%= helper.upperFirst(msg) %>>;<% }); %>
+<% }); %>

packages/common-cosmos/src/codegen/util.ts

@@ -0,0 +1,14 @@
+// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+
+import {parseCosmosProjectManifest} from '../project/load';
+
+export function validateCosmosManifest(manifest: {
+  network: {chainTypes?: Map<string, {file: string; messages: string[]}>};
+}): boolean {
+  try {
+    return !!parseCosmosProjectManifest(manifest);
+  } catch (e) {
+    return false;
+  }
+}

packages/common-cosmos/src/index.ts

@@ -1,4 +1,5 @@
 // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
+export * from './codegen';
 export * from './project';

packages/common-cosmos/src/project/types.ts

@@ -3,6 +3,7 @@
 
 import {IProjectManifest, ProjectNetworkConfig} from '@subql/common';
 import {SubqlCosmosDatasource, CustomModule} from '@subql/types-cosmos';
+import {Root} from 'protobufjs';
 
 // All of these used to be redefined in this file, re-exporting for simplicity
 export {
@@ -27,3 +28,13 @@ export interface CosmosProjectNetworkConfig extends ProjectNetworkConfig {
   chainId?: string;
   bypassBlocks?: (number | string)[];
 }
+
+export type CosmosChainType = CustomModule & {
+  proto: Root;
+  packageName?: string;
+};
+
+export type CosmosProjectNetConfig = CosmosProjectNetworkConfig & {
+  chainId: string;
+  chainTypes: Map<string, CosmosChainType> & {protoRoot: protobuf.Root};
+};