Merge pull request #120 from superfaceai/feat/typescript_profiles

feat: support TypeScript profiles

Makefile

@@ -144,13 +144,14 @@ clean_core_js:
 ##############
 build_packages: build_python_host build_nodejs_host build_cfw_host
 deps_packages: deps_python_host deps_nodejs_host deps_cfw_host
+test_packages: test_nodejs_host test_cfw_host test_python_host
 
 # Node.js Host
 deps_nodejs_host:
 	cd packages/nodejs_host && yarn install
 build_nodejs_host: deps_nodejs_host ${CORE_ASYNCIFY_WASM}
 	mkdir -p ${NODEJS_HOST_ASSETS}
-	cp ${CORE_ASYNCIFY_WASM} ${NODEJS_HOST_ASSETS}/core-async.wasm # copy wasm always because cached docker artifacts can have older timestamp
+	cp ${CORE_ASYNCIFY_WASM} ${NODEJS_HOST_ASSETS}/core-async.wasm
 	cd packages/nodejs_host && yarn build	
 test_nodejs_host: build_nodejs_host ${TEST_CORE_ASYNCIFY_WASM}
 	cp ${TEST_CORE_ASYNCIFY_WASM} ${NODEJS_HOST_ASSETS}/test-core-async.wasm
@@ -161,7 +162,7 @@ deps_cfw_host:
 	cd packages/cloudflare_worker_host && yarn install
 build_cfw_host: deps_cfw_host ${CORE_ASYNCIFY_WASM}
 	mkdir -p ${CFW_HOST_ASSETS}
-	cp ${CORE_ASYNCIFY_WASM} ${CFW_HOST_ASSETS}/core-async.wasm # copy wasm always because cached docker artifacts can have older timestamp
+	cp ${CORE_ASYNCIFY_WASM} ${CFW_HOST_ASSETS}/core-async.wasm
 	cd packages/cloudflare_worker_host && yarn build	
 test_cfw_host: build_cfw_host ${TEST_CORE_ASYNCIFY_WASM}
 	cp ${TEST_CORE_ASYNCIFY_WASM} ${CFW_HOST_ASSETS}/test-core-async.wasm

core/core/src/sf_core/metrics.rs

@@ -28,26 +28,29 @@ impl<'a> PerformMetricsData<'a> {
     pub fn get_profile(&self) -> Cow<'_, str> {
         match self.profile {
             Some(profile) => Cow::Borrowed(profile),
-            None => Cow::Owned(
-                self.profile_url
-                    .split('/')
-                    .last()
-                    .and_then(|b| b.strip_suffix(".profile"))
-                    .unwrap()
-                    .replace('.', "/"),
-            ),
+            None => match self.profile_url.split('/').last() {
+                // treat anything from .profile until the end of the url as an extension
+                // this works for both .profile and .profile.ts extensions
+                Some(last_segment) => match last_segment.rfind(".profile") {
+                    Some(ext_start) => Cow::Owned(last_segment[..ext_start].replace('.', "/")),
+                    None => Cow::Borrowed("unknown"),
+                },
+                None => Cow::Borrowed("unknown"),
+            },
         }
     }
 
     pub fn get_provider(&self) -> &str {
         match self.provider {
             Some(provider) => provider,
-            None => self
-                .provider_url
-                .split('/')
-                .last()
-                .and_then(|b| b.strip_suffix(".provider.json"))
-                .unwrap(),
+            None => match self.profile_url.split('/').last() {
+                // treat anything from .provider until the end of the url as an extension
+                Some(last_segment) => match last_segment.rfind(".provider") {
+                    Some(ext_start) => &last_segment[..ext_start],
+                    None => "unknown",
+                },
+                None => "unknown",
+            },
         }
     }
 }

examples/cloudflare_worker/src/comlinks.ts

@@ -15,7 +15,7 @@ import mapSendSms from '../superface/communication.send-sms.twilio.map.js';
 import providerTwilio from '../superface/twilio.provider.json';
 
 // @ts-ignore
-import profileExample from '../superface/wasm-sdk.example.profile';
+import profileExample from '../superface/wasm-sdk.example.profile.ts';
 // @ts-ignore
 import mapExample from '../superface/wasm-sdk.example.localhost.map.js';
 // @ts-ignore
@@ -28,7 +28,7 @@ export const COMLINK_IMPORTS = {
   'superface/communication.send-sms.profile': new Uint8Array(profileSendSms),
   'superface/communication.send-sms.twilio.map.js': new Uint8Array(mapSendSms),
   'superface/twilio.provider.json': new Uint8Array(providerTwilio as any),
-  'superface/wasm-sdk.example.profile': new Uint8Array(profileExample),
+  'superface/wasm-sdk.example.profile.ts': new Uint8Array(profileExample),
   'superface/wasm-sdk.example.localhost.map.js': new Uint8Array(mapExample),
   'superface/localhost.provider.json': new Uint8Array(providerLocalhost as any)
 };

examples/comlinks/src/wasm-sdk.example.localhost.map.js

@@ -1,14 +1,14 @@
 /// <reference types="@superface/map-std" />
+/// <reference path="./wasm-sdk.example.profile.ts" />
 // @ts-check
 
 const manifest = {
   profile: 'wasm-sdk/[email protected]',
   provider: 'localhost'
 };
 
-// var to hoist it like a function would be
-// can't be a function because then the type annotation doesn't work
-/** @type {Usecase<{ safety: 'safe', input: { id: AnyValue }, result: { url: AnyValue, method: AnyValue, query: AnyValue, headers: AnyValue }, error: { title: string, detail: string } }>} */
+// var to hoist it like a function would be - can't be a function because then the type annotation doesn't work
+/** @type {Example} */
 var Example = ({ input, parameters, services }) => {
   // @ts-ignore
   __ffi.unstable.printDebug('Input:', input);

examples/comlinks/src/wasm-sdk.example.profile.ts

@@ -0,0 +1,7 @@
+/// <reference types="@superface/map-std" />
+type Example = Usecase<{
+  safety: 'safe'
+  input: { id: AnyValue }
+  result: { url: AnyValue, method: AnyValue, query: AnyValue, headers: AnyValue }
+  error: { title: AnyValue, detail?: AnyValue }
+}>;

examples/run.sh

@@ -22,7 +22,7 @@ case $1 in
 			make build_cfw_host $MAKE_FLAGS
 		fi
 		cd "$base/cloudflare_worker"
-		yarn install
+		yarn install --force
 		yarn dev
 	;;
 

packages/cloudflare_worker_host/src/index.ts

@@ -31,6 +31,10 @@ class CfwFileSystem implements FileSystem {
     this.files = new HandleMap();
   }
 
+  async exists(path: string): Promise<boolean> {
+    return this.preopens[path] !== undefined;
+  }
+
   async open(path: string, options: { createNew?: boolean, create?: boolean, truncate?: boolean, append?: boolean, write?: boolean, read?: boolean }): Promise<number> {
     if (options.read !== true) {
       throw new WasiError(WasiErrno.EROFS);
@@ -88,7 +92,11 @@ class CfwNetwork implements Network {
     try {
       response = await fetch(input, init);
     } catch (err: unknown) {
-      throw err; // TODO: are there any errors that we need to handle here?
+      if (typeof err === 'object' && err !== null && 'message' in err) {
+        // found a `Error: Network connection lost` caused by `kj/async-io-unix.c++:186: disconnected` in the wild
+        throw new HostError(ErrorCode.NetworkError, `${err.message}`);
+      }
+      throw err;
     }
 
     if (response.status === 530) {
@@ -275,10 +283,12 @@ export type ClientPerformOptions = {
 class InternalClient {
   private readonly app: App;
   private ready = false;
+  private readonly fileSystem: CfwFileSystem;
 
   constructor(readonly options: ClientOptions = {}) {
+    this.fileSystem = new CfwFileSystem(options.preopens ?? {});
     this.app = new App({
-      fileSystem: new CfwFileSystem(options.preopens ?? {}),
+      fileSystem: this.fileSystem,
       textCoder: new CfwTextCoder(),
       timers: new CfwTimers(),
       network: new CfwNetwork(),
@@ -324,9 +334,20 @@ class InternalClient {
     const resolvedProfile = profile.replace(/\//g, '.'); // TODO: be smarter about this
     const assetsPath = this.options.assetsPath ?? 'superface'; // TODO: path join? - not sure if we are going to stick with this VFS
 
+    let profilePath = `${assetsPath}/${resolvedProfile}.profile.ts`;
+    // migration from Comlink to TypeScript profiles
+    const profilePathComlink = `${assetsPath}/${resolvedProfile}.profile`;
+    if (
+      !(await this.fileSystem.exists(profilePath))
+      && (await this.fileSystem.exists(profilePathComlink))
+    ) {
+      profilePath = profilePathComlink;
+    }
+    profilePath = `file://${profilePath}`;
+
     try {
       return await this.app.perform(
-        `file://${assetsPath}/${resolvedProfile}.profile`,
+        profilePath,
         `file://${assetsPath}/${provider}.provider.json`,
         `file://${assetsPath}/${resolvedProfile}.${provider}.map.js`,
         usecase,

packages/javascript_common/src/app.ts

@@ -1,108 +1,9 @@
-import { Asyncify } from './asyncify.js';
+import type { SecurityValuesMap } from './security.js';
+import type { AppContext, FileSystem, Network, Persistence, TextCoder, Timers, WasiContext } from './interfaces.js';
 import { PerformError, UnexpectedError, UninitializedError, ValidationError, WasiErrno, WasiError } from './error.js';
-import { HandleMap } from './handle_map.js';
-import { AppContext, FileSystem, Network, Persistence, TextCoder, Timers, WasiContext } from './interfaces.js';
-import { SecurityValuesMap } from './security.js';
+import { AsyncMutex, Asyncify, HandleMap, ReadableStreamAdapter, Stream } from './lib/index.js';
 import * as sf_host from './sf_host.js';
 
-class ReadableStreamAdapter implements Stream {
-  private chunks: Uint8Array[];
-  private readonly reader?: ReadableStreamDefaultReader<Uint8Array>;
-  constructor(stream: ReadableStream<Uint8Array> | null) {
-    this.reader = stream?.getReader();
-    this.chunks = [];
-  }
-  async read(out: Uint8Array): Promise<number> {
-    if (this.reader === undefined) {
-      return 0;
-    }
-
-    if (this.chunks.length === 0) {
-      const readResult = await this.reader.read();
-      if (readResult.value === undefined) {
-        return 0;
-      }
-
-      this.chunks.push(readResult.value);
-    }
-
-    // TODO: coalesce multiple smaller chunks into one read
-    let chunk = this.chunks.shift()!;
-    if (chunk.byteLength > out.byteLength) {
-      const remaining = chunk.subarray(out.byteLength);
-      chunk = chunk.subarray(0, out.byteLength);
-
-      this.chunks.unshift(remaining);
-    }
-
-    const count = Math.min(chunk.byteLength, out.byteLength);
-    for (let i = 0; i < count; i += 1) {
-      out[i] = chunk[i];
-    }
-
-    return count;
-  }
-  async write(data: Uint8Array): Promise<number> {
-    throw new Error('not implemented');
-  }
-  async close(): Promise<void> {
-    // TODO: what to do here?
-  }
-}
-
-/** Async mutex allows us to synchronize multiple async tasks.
- * 
- * For example, if a perform is in-flight but is waiting for I/O the async task is suspended. If at the same time
- * the periodic timer fires, this could cause core to be invoked twice within the same asyncify context, causing undefined behavior.
- * 
- * We can avoid this by synchronizing over core.
- * 
- * Note that this is not thread safe (concurrent), but merely task safe (asynchronous).
- */
-export class AsyncMutex<T> {
-  /** Promise to be awaited to synchronize between tasks. */
-  private condvar: Promise<void>;
-  /** Indicator of whether the mutex is currently locked. */
-  private isLocked: boolean;
-  private value: T;
-
-  constructor(value: T) {
-    this.condvar = Promise.resolve();
-    this.isLocked = false;
-    this.value = value;
-  }
-
-  /**
-   * Get the protected value without respecting the lock.
-   * 
-   * This is unsafe, but it is needed to get access to memory in sf_host imports.
-   */
-  get unsafeValue(): T {
-    return this.value;
-  }
-
-  public async withLock<R>(fn: (value: T) => R): Promise<Awaited<R>> {
-    do {
-      // Under the assumption that we do not have concurrency it can never happen that two tasks
-      // pass over the condition of this loop and think they both have a lock - that would imply there exists task preemption in synchronous code.
-      //
-      // If there ever is threading or task preemption, we will need to use other means (atomics, spinlocks).
-      await this.condvar;
-    } while (this.isLocked);
-
-    this.isLocked = true;
-    let notify: () => void;
-    this.condvar = new Promise((resolve) => { notify = resolve; });
-
-    try {
-      return await fn(this.value);
-    } finally {
-      this.isLocked = false;
-      notify!();
-    }
-  }
-}
-
 function headersToMultimap(headers: Headers): Record<string, string[]> {
   const result: Record<string, string[]> = {};
 
@@ -118,14 +19,6 @@ function headersToMultimap(headers: Headers): Record<string, string[]> {
   return result;
 }
 
-type Stream = {
-  /** Reads up to `out.length` bytes from the stream, returns number of bytes read or throws a `WasiError`. */
-  read(out: Uint8Array): Promise<number>;
-  /** Writes up to `data.length` bytes into the stream, returns number of bytes written or throws a `WasiError`. */
-  write(data: Uint8Array): Promise<number>;
-  /** Closes the stream, returns undefined or throws a `WasiError`. */
-  close(): Promise<void>;
-};
 type AppCore = {
   instance: WebAssembly.Instance;
   asyncify: Asyncify;

packages/javascript_common/src/index.ts

@@ -1,6 +1,5 @@
-export { App, AsyncMutex } from './app.js';
+export { App } from './app.js';
 export * from './error.js';
-export { HandleMap } from './handle_map.js';
 export type { FileSystem, Network, Persistence, TextCoder, Timers, WasiContext } from './interfaces.js';
-export { SecurityValuesMap } from './security.js';
-export * from './wasm.js';
+export type { SecurityValuesMap } from './security.js';
+export { HandleMap, AsyncMutex, corePathURL } from './lib/index.js';

packages/javascript_common/src/interfaces.ts

@@ -8,6 +8,8 @@ export interface AppContext {
   closeStream(handle: number): Promise<void>;
 }
 export interface FileSystem {
+  /** Return true if the file exists (can be `stat`ed). */
+  exists(path: string): Promise<boolean>;
   open(path: string, options: { createNew?: boolean, create?: boolean, truncate?: boolean, append?: boolean, write?: boolean, read?: boolean }): Promise<number>;
   /** Read bytes and write them to `out`. Returns number of bytes read. */
   read(handle: number, out: Uint8Array): Promise<number>;