PDE-4968 feat(schema-to-ts) - Introduce the Schema-to-TS compiler (#818)

* Add schema-to-ts compiler as new package

* Add commands and scripts to generate typings

* Format compiler with repo's prettier (i.e single quotes)

* Remove ZPC and ZPS dependencies from schema-to-ts

* Improve compiler to be better DevPlatform citizen

This improves the CLI of the schema-compiler to use the input json and
output typescript to sensible defaults within the repo, and makes a
bunch of changes that improve the logging, command line options and
features, and improves the quality of the final output typescript.

* Abstract zpc types to module to include code-gen

This moves the `index.d.ts` into a `types/` directory and renames it to
zapier.custom.d.ts. Then a new `index.d.ts` file has been added, that
imports from both this hand-written new module, and the generated
sibling module, unifying the imports.
With any luck, this commit should include the generated typings because
of the pre-commit hooks.

* Improve compiler command line arguments

* Abstract snippets with a builder function

* Pre-PR cleanup & commenting

* Switch to single quote formatting for schema types

* Extract compiler logic from CLI to main.ts for easier testing

* Move PerformFunction definition to zapier.custom.d.ts

* Remove serialised code formats for FunctionSchema, alias PerformFunction

* Don't generate PerformFunction with compiler, use from custom types

* Bump to json-schema-to-typescript fro, v13 -> v14

This also pins the version, and updates (just a type) imported from the
swapped out @bcherny/json-schema-ref-parser repo to
@apidevtools/json-schema-ref-parser

* Move schema-to-ts to root

The root was chosen instead of ./tools or such because there are no
other tools currently.

* Adjust yarn commands for new schema-to-ts location

* Add husky commands to build schema types

* Add README for schema-to-ts

* Update code generation comments

* Add generated typescript tests

* Add more support for app middleware functions

* Fix more CI testing for typegeneration

* Fix TS version used in templates & smoke tests

Because this uses `export type` syntax, version 5 of TS is needed.

* Bump schema-to-ts to TypeScript 5.5.3

* Revert yarn.lock to main branch

* Add @types/node for schema-to-ts

* Revert yarn.lock to main branch

* Add post-install hooks to core and schema for TS compilation

Based on Raúl's advice, so that the schema-to-ts tool is ready after a
standard `yarn` invocation even from the root to install dependencies.

* Revert "Add post-install hooks to core and schema for TS compilation"

This reverts commit beca90f.

* Add CI job for testing schema-to-ts

* change yarn working directory in CI to schema-to-ts

.github/workflows/ci.yaml

@@ -50,6 +50,24 @@ jobs:
         run: yarn test
         working-directory: packages/legacy-scripting-runner
 
+  test-schema-to-ts:
+    name: Test - schema-to-ts
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+          cache: yarn
+      - name: Install dependencies
+        run: yarn --frozen-lockfile
+        working-directory: schema-to-ts
+      - name: Test - schema-to-ts
+        run: yarn test
+        working-directory: schema-to-ts
+
   test-cli:
     name: Test - cli
     runs-on: ${{ matrix.os }}

.husky/pre-commit

@@ -1,4 +1,4 @@
 #!/bin/sh
 . "$(dirname "$0")/_/husky.sh"
 
-yarn lerna run --stream precommit && yarn lint-staged
+yarn lerna run --stream precommit && yarn generate-types && yarn lint-staged

example-apps/typescript/package.json

@@ -19,7 +19,7 @@
     "@types/node": "^13.13.5",
     "@types/node-fetch": "^2.6.11",
     "rimraf": "^3.0.2",
-    "typescript": "^4.9.4"
+    "typescript": "^5.5.3"
   },
   "private": true
 }

example-apps/typescript/tsconfig.json

@@ -1,11 +1,12 @@
 {
   "compilerOptions": {
-    "target": "es2019",
+    "target": "ESNext",
     "module": "commonjs",
     "moduleResolution": "node",
     "lib": ["esnext"],
     "outDir": "./lib",
     "rootDir": "./src",
-    "strict": true
+    "strict": true,
+    "skipLibCheck": true
   }
 }

package.json

@@ -25,18 +25,19 @@
     "packages/*"
   ],
   "scripts": {
-    "test": "yarn workspaces run test",
+    "test": "yarn workspaces run test && (cd schema-to-ts && yarn test)",
     "smoke-test": "yarn workspace zapier-platform-cli run smoke-test && yarn workspace zapier-platform-core run smoke-test && yarn workspace zapier-platform-schema run smoke-test",
     "lint": "lerna run lint",
     "lint:fix": "lerna run lint:fix",
     "lint-examples": "eslint examples",
     "validate": "lerna run validate",
     "bump": "./scripts/bump.js",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "generate-types": "cd schema-to-ts && yarn generate-types:build"
   },
   "husky": {
     "hooks": {
-      "pre-commit": "lerna run --stream precommit && lint-staged"
+      "pre-commit": "lerna run --stream precommit && yarn generate-types && lint-staged"
     }
   },
   "lint-staged": {

packages/cli/src/generators/index.js

@@ -139,7 +139,7 @@ const writeForStandaloneTemplate = (gen) => {
         '@types/node': '^14',
         '@types/node-fetch': '^2.6.11',
         rimraf: '^3.0.2',
-        typescript: '^4.9.4',
+        typescript: '5.5.3',
       },
     },
   }[gen.options.template];

packages/core/package.json

@@ -7,20 +7,21 @@
   "author": "Zapier Engineering <[email protected]>",
   "license": "SEE LICENSE IN LICENSE",
   "main": "index.js",
-  "typings": "index.d.ts",
+  "types": "types/index.d.ts",
   "files": [
     "/include/",
-    "/index.d.ts",
     "/index.js",
-    "/src/"
+    "/src/",
+    "/types/"
   ],
   "scripts": {
     "preversion": "git pull && yarn test",
     "version": "node bin/bump-dependencies.js && yarn && git add package.json yarn.lock",
     "postversion": "git push && git push --tags",
     "main-tests": "mocha -t 20s --recursive test --exit",
+    "type-tests": "tsd",
     "solo-test": "test $(OPT_OUT_PATCH_TEST_ONLY=yes mocha --recursive test -g 'should be able to opt out of patch' -R json | jq '.stats.passes') -eq 1 && echo 'Ran 1 test and it passed!'",
-    "test": "yarn main-tests && yarn solo-test",
+    "test": "yarn main-tests && yarn solo-test && yarn type-tests",
     "test:debug": "mocha inspect -t 10s --recursive test",
     "debug": "mocha -t 10s --inspect-brk --recursive test",
     "test:w": "mocha -t 10s --recursive test --watch",
@@ -61,7 +62,8 @@
     "dicer": "^0.3.1",
     "fs-extra": "^11.1.1",
     "mock-fs": "^5.2.0",
-    "nock": "^13.5.4"
+    "nock": "^13.5.4",
+    "tsd": "^0.31.1"
   },
   "optionalDependencies": {
     "@types/node": "^20.3.1"

packages/core/types/index.d.ts

@@ -0,0 +1,2 @@
+export type * from './zapier.generated';
+export * from './zapier.custom';

packages/core/types/index.test-d.ts

@@ -0,0 +1,152 @@
+import type {
+  AfterResponseMiddleware,
+  BeforeRequestMiddleware,
+  Bundle,
+  PerformFunction,
+  ZObject,
+} from './zapier.custom';
+import type {
+  App,
+  Authentication,
+  AuthenticationOAuth2Config,
+  BasicActionOperation,
+  BasicCreateActionOperation,
+  BasicDisplay,
+  BasicHookOperation,
+  BasicPollingOperation,
+  Create,
+  Search,
+  Trigger,
+} from './zapier.generated';
+
+import { expectType } from 'tsd';
+
+const basicDisplay: BasicDisplay = {
+  label: 'some-label',
+  description: 'some-description',
+  directions: 'some-directions',
+  hidden: false,
+};
+expectType<BasicDisplay>(basicDisplay);
+
+const oauth2Config: AuthenticationOAuth2Config = {
+  authorizeUrl: 'https://example.com/authorize',
+  getAccessToken: 'https://example.com/token',
+  refreshAccessToken: async (z: ZObject, b: Bundle) => 'some-refresh-token',
+};
+expectType<AuthenticationOAuth2Config>(oauth2Config);
+
+const authentication: Authentication = {
+  type: 'oauth2',
+  test: async (z: ZObject, b: Bundle) => ({ data: true }),
+  oauth2Config,
+};
+expectType<Authentication>(authentication);
+
+const createOperation: BasicCreateActionOperation = {
+  inputFields: [{ key: 'some-input-key-1', type: 'string', required: true }],
+  perform: async (z: ZObject, b: Bundle) => ({ data: true }),
+  sample: { id: 'some-id', name: 'some-name' },
+};
+expectType<BasicCreateActionOperation>(createOperation);
+
+const create: Create = {
+  key: 'some_create_key_v1',
+  noun: 'Some Noun',
+  display: {
+    label: 'some create label',
+    description: 'some trigger description',
+  },
+  operation: createOperation,
+};
+expectType<Create>(create);
+
+const pollingOperation: BasicPollingOperation = {
+  type: 'polling',
+  inputFields: [{ key: 'some-input-key-1', type: 'number', required: true }],
+  perform: async (z: ZObject, b: Bundle) => ({ data: true }),
+};
+
+const pollingTrigger: Trigger = {
+  key: 'some_polling_trigger_key_v1',
+  noun: 'Some Noun',
+  display: {
+    label: 'some polling trigger label',
+    description: 'some polling trigger description',
+  },
+  operation: pollingOperation,
+};
+expectType<Trigger>(pollingTrigger);
+
+const hookOperation: BasicHookOperation = {
+  type: 'hook',
+  inputFields: [{ key: 'some-input-key-1', type: 'boolean', required: false }],
+  perform: async (z: ZObject, b: Bundle) => ({ data: true }),
+  performList: async (z: ZObject, b: Bundle) => [{ data: true }],
+  performSubscribe: async (z: ZObject, b: Bundle) => ({ data: true }),
+  performUnsubscribe: async (z: ZObject, b: Bundle) => ({ data: true }),
+};
+expectType<BasicHookOperation>(hookOperation);
+
+const hookTrigger: Trigger = {
+  key: 'some_hook_trigger_key_v1',
+  noun: 'Some Noun',
+  display: {
+    label: 'some hook label',
+    description: 'some hook description',
+  },
+  operation: hookOperation,
+};
+expectType<Trigger>(hookTrigger);
+
+const searchOperation: BasicActionOperation = {
+  inputFields: [{ key: 'some-input-key-1', type: 'file', required: true }],
+  perform: async (z: ZObject, b: Bundle) => [{ data: true }],
+};
+expectType<BasicActionOperation>(searchOperation);
+
+const search: Search = {
+  key: 'some_search_key_v1',
+  noun: 'Some Noun',
+  display: {
+    label: 'some search label',
+    description: 'some search description',
+  },
+  operation: searchOperation,
+};
+expectType<Search>(search);
+
+const addBearerHeader: BeforeRequestMiddleware = (request, z, bundle) => {
+  if (bundle?.authData?.access_token && !request.headers!.Authorization) {
+    request.headers!.Authorization = `Bearer ${bundle.authData.access_token}`;
+  }
+  return request;
+};
+expectType<BeforeRequestMiddleware>(addBearerHeader);
+
+const checkPermissionsError: AfterResponseMiddleware = (response, z) => {
+  if (response.status === 403) {
+    throw new z.errors.Error(
+      response.json?.['o:errorDetails']?.[0].detail,
+      response.status.toString()
+    );
+  }
+  return response;
+};
+expectType<AfterResponseMiddleware>(checkPermissionsError);
+
+const app: App = {
+  platformVersion: '0.0.1',
+  version: '0.0.1',
+
+  beforeRequest: [addBearerHeader],
+  afterResponse: [checkPermissionsError],
+
+  creates: { [create.key]: create },
+  triggers: {
+    [pollingTrigger.key]: pollingTrigger,
+    [hookTrigger.key]: hookTrigger,
+  },
+  searches: { [search.key]: search },
+};
+expectType<App>(app);

packages/core/index.d.ts → packages/core/types/zapier.custom.d.ts

@@ -22,10 +22,6 @@ export const createAppTester: (
   clearZcacheBeforeUse?: boolean
 ) => Promise<T>; // appTester always returns a promise
 
-// internal only
-// export const integrationTestHandler: () => any;
-// export const createAppHandler: (appRaw: object) => any
-
 type HttpMethod =
   | 'GET'
   | 'POST'
@@ -118,9 +114,9 @@ export interface HttpResponse extends BaseHttpResponse {
 
 export interface RawHttpResponse extends BaseHttpResponse {
   body: NodeJS.ReadableStream;
-	buffer(): Promise<Buffer>;
-	json(): Promise<object>;
-	text(): Promise<string>;
+  buffer(): Promise<Buffer>;
+  json(): Promise<object>;
+  text(): Promise<string>;
 }
 
 type DehydrateFunc = <T>(
@@ -204,3 +200,26 @@ export interface ZObject {
     delete: (key: string) => Promise<boolean>;
   };
 }
+
+/**
+ * A function that performs the action.
+ *
+ * @template BI The shape of data in the `bundle.inputData` object.
+ * @template R The return type of the function.
+ */
+export type PerformFunction<BI = Record<string, any>, R = any> = (
+  z: ZObject,
+  bundle: Bundle<BI>
+) => Promise<R>;
+
+export type BeforeRequestMiddleware = (
+  request: HttpRequestOptions,
+  z?: ZObject,
+  bundle?: Bundle
+) => HttpRequestOptions;
+
+export type AfterResponseMiddleware = (
+  response: HttpResponse,
+  z: ZObject,
+  bundle?: Bundle
+) => HttpResponse;