*: upgrade to ES2022 modules

Closes: #632

Dockerfile

@@ -8,7 +8,7 @@ COPY server/package.json server/
 RUN npm ci
 
 COPY isomorphic-wrtc/ isomorphic-wrtc/
-COPY discojs/tsconfig.base.json discojs/
+COPY tsconfig.base.json .
 COPY discojs/discojs-core/ discojs/discojs-core/
 COPY discojs/discojs-node/ discojs/discojs-node/
 RUN npm run build --workspace=@epfml/discojs-core --workspace=@epfml/discojs-node

cli/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@epfml/disco-cli",
   "private": true,
+  "type": "module",
   "main": "dist/cli.js",
   "scripts": {
     "watch": "nodemon --ext ts --ignore dist --watch ../discojs/discojs-node/dist --watch ../server/dist --watch . --exec npm run",

cli/src/args.ts

@@ -1,5 +1,6 @@
 import { parse } from 'ts-command-line-args'
 import { Map } from 'immutable'
+
 import type { Task } from '@epfml/discojs-core'
 import { defaultTasks } from '@epfml/discojs-core'
 

cli/src/cli.ts

@@ -4,9 +4,9 @@ import type { TrainerLog, data, Task } from '@epfml/discojs-core'
 import { Disco, aggregator as aggregators, client as clients } from '@epfml/discojs-core'
 import { startServer } from '@epfml/disco-server'
 
-import { saveLog } from './utils'
-import { getTaskData } from './data'
-import { args } from './args'
+import { saveLog } from './utils.js'
+import { getTaskData } from './data.js'
+import { args } from './args.js'
 
 async function runUser (task: Task, url: URL, data: data.DataSplit): Promise<TrainerLog> {
   const client = new clients.federated.FederatedClient(url, task, new aggregators.MeanAggregator())

cli/tsconfig.json

@@ -1,19 +1,5 @@
 {
-  "compilerOptions": {
-    "target": "ES5",
-    "lib": ["ES5"],
-
-    "strict": true,
-
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "moduleResolution": "node",
-    "downlevelIteration": true, // needed by <ES6
-
-    "declaration": true,
-
-    "baseUrl": "src",
-    "outDir":  "dist",
-  },
+  "extends": "../tsconfig.base.json",
+  "compilerOptions": { "outDir":  "dist" },
   "include": ["src"],
 }

discojs/discojs-core/.mocharc.json

@@ -2,5 +2,6 @@
   "spec": ["src"],
   "recursive": true,
   "extension": ["ts"],
+  "loader": "ts-node/esm",
   "require": "ts-node/register"
 }

discojs/discojs-core/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@epfml/discojs-core",
   "version": "2.1.1",
+  "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {

discojs/discojs-core/src/aggregator/base.ts

@@ -1,8 +1,8 @@
 import { Map, Set } from 'immutable'
 
-import type { client, Model, AsyncInformant } from '..'
+import type { client, Model, AsyncInformant } from '../index.js'
 
-import { EventEmitter } from '../utils/event_emitter'
+import { EventEmitter } from '../utils/event_emitter.js'
 
 export enum AggregationStep {
   ADD,

discojs/discojs-core/src/aggregator/get.ts

@@ -1,4 +1,5 @@
-import { aggregator, type Task } from '..'
+import type { Task } from '../index.js'
+import { aggregator } from '../index.js'
 
 /**
  * Enumeration of the available types of aggregator.

discojs/discojs-core/src/aggregator/index.ts

@@ -1,11 +1,11 @@
-import type { WeightsContainer } from '../weights'
-import type { Base } from './base'
+import type { WeightsContainer } from '../weights/index.js'
+import type { Base } from './base.js'
 
-export { Base as AggregatorBase, AggregationStep } from './base'
-export { MeanAggregator } from './mean'
-export { RobustAggregator } from './robust'
-export { SecureAggregator } from './secure'
+export { Base as AggregatorBase, AggregationStep } from './base.js'
+export { MeanAggregator } from './mean.js'
+export { RobustAggregator } from './robust.js'
+export { SecureAggregator } from './secure.js'
 
-export { getAggregator, AggregatorChoice } from './get'
+export { getAggregator, AggregatorChoice } from './get.js'
 
 export type Aggregator = Base<WeightsContainer>

discojs/discojs-core/src/aggregator/mean.spec.ts

@@ -1,9 +1,9 @@
 import { assert, expect } from 'chai'
 import type { Map } from 'immutable'
 
-import type { client, Model } from '..'
-import { aggregator, defaultTasks } from '..'
-import { AggregationStep } from './base'
+import type { client, Model } from '../index.js'
+import { aggregator, defaultTasks } from '../index.js'
+import { AggregationStep } from './base.js'
 
 const model = defaultTasks.titanic.getModel()
 const id = 'a'

discojs/discojs-core/src/aggregator/mean.ts

@@ -1,8 +1,8 @@
 import type { Map } from 'immutable'
 
-import { AggregationStep, Base as Aggregator } from './base'
-import type { Model, WeightsContainer, client } from '..'
-import { aggregation } from '..'
+import { AggregationStep, Base as Aggregator } from './base.js'
+import type { Model, WeightsContainer, client } from '../index.js'
+import { aggregation } from '../index.js'
 
 /**
  * Mean aggregator whose aggregation step consists in computing the mean of the received weights.

discojs/discojs-core/src/aggregator/robust.ts

@@ -1,5 +1,5 @@
-import { Base as Aggregator } from './base'
-import type { client, Model, WeightsContainer } from '..'
+import { Base as Aggregator } from './base.js'
+import type { client, Model, WeightsContainer } from '../index.js'
 
 import type { Map } from 'immutable'
 

discojs/discojs-core/src/aggregator/secure.spec.ts

@@ -1,7 +1,7 @@
 import { List, Set, Range } from 'immutable'
 import { assert } from 'chai'
 
-import { aggregator as aggregators, aggregation, WeightsContainer } from '@epfml/discojs-core'
+import { aggregator as aggregators, aggregation, WeightsContainer } from '../index.js'
 
 describe('secret shares test', function () {
   const epsilon = 1e-4

discojs/discojs-core/src/aggregator/secure.ts

@@ -1,10 +1,9 @@
-import * as crypto from 'crypto'
 import { Map, List, Range } from 'immutable'
-import tf from '@tensorflow/tfjs'
+import * as tf from '@tensorflow/tfjs'
 
-import { AggregationStep, Base as Aggregator } from './base'
-import type { Model, WeightsContainer, client } from '..'
-import { aggregation } from '..'
+import { AggregationStep, Base as Aggregator } from './base.js'
+import type { Model, WeightsContainer, client } from '../index.js'
+import { aggregation } from '../index.js'
 
 /**
  * Aggregator implementing secure multi-party computation for decentralized learning.
@@ -14,8 +13,6 @@ import { aggregation } from '..'
  * Finally, nodes are able to average the received partial sums to establish the aggregation result.
  */
 export class SecureAggregator extends Aggregator<WeightsContainer> {
-  public static readonly MAX_SEED: number = 2 ** 47
-
   constructor (
     model?: Model,
     private readonly maxShareValue = 100
@@ -93,7 +90,11 @@ export class SecureAggregator extends Aggregator<WeightsContainer> {
    * a uniform distribution between (-maxShareValue, maxShareValue).
    */
   public generateRandomShare (secret: WeightsContainer): WeightsContainer {
-    const seed = crypto.randomInt(SecureAggregator.MAX_SEED)
+    const MAX_SEED_BITS = 47
+
+    const random = crypto.getRandomValues(new BigInt64Array(1))[0]
+    const seed = Number(BigInt.asUintN(MAX_SEED_BITS, random))
+
     return secret.map((t) =>
       tf.randomUniform(t.shape, -this.maxShareValue, this.maxShareValue, 'float32', seed))
   }

discojs/discojs-core/src/async_informant.ts

@@ -1,4 +1,4 @@
-import type { AggregatorBase } from './aggregator'
+import type { AggregatorBase } from './aggregator/index.js'
 
 export class AsyncInformant<T> {
   private _round = 0

discojs/discojs-core/src/client/base.ts

@@ -1,11 +1,11 @@
 import axios from 'axios'
 import type { Set } from 'immutable'
 
-import type { Model, Task, TrainingInformant, WeightsContainer } from '..'
-import { serialization } from '..'
-import type { NodeID } from './types'
-import type { EventConnection } from './event_connection'
-import type { Aggregator } from '../aggregator'
+import type { Model, Task, TrainingInformant, WeightsContainer } from '../index.js'
+import { serialization } from '../index.js'
+import type { NodeID } from './types.js'
+import type { EventConnection } from './event_connection.js'
+import type { Aggregator } from '../aggregator/index.js'
 
 /**
  * Main, abstract, class representing a Disco client in a network, which handles

discojs/discojs-core/src/client/decentralized/base.ts

@@ -1,12 +1,12 @@
 import { Map, Set } from 'immutable'
 
-import { type TrainingInformant, type WeightsContainer, serialization } from '../..'
-import { Client, type NodeID } from '..'
-import { type, type ClientConnected } from '../messages'
-import { timeout } from '../utils'
-import { type EventConnection, WebSocketServer, waitMessage, type PeerConnection, waitMessageWithTimeout } from '../event_connection'
-import { PeerPool } from './peer_pool'
-import * as messages from './messages'
+import { type TrainingInformant, type WeightsContainer, serialization } from '../../index.js'
+import { Client, type NodeID } from '../index.js'
+import { type, type ClientConnected } from '../messages.js'
+import { timeout } from '../utils.js'
+import { type EventConnection, WebSocketServer, waitMessage, type PeerConnection, waitMessageWithTimeout } from '../event_connection.js'
+import { PeerPool } from './peer_pool.js'
+import * as messages from './messages.js'
 
 /**
  * Represents a decentralized client in a network of peers. Peers coordinate each other with the

discojs/discojs-core/src/client/decentralized/index.ts

@@ -1,2 +1,2 @@
-export { Base as DecentralizedClient } from './base'
-export * as messages from './messages'
+export { Base as DecentralizedClient } from './base.js'
+export * as messages from './messages.js'

discojs/discojs-core/src/client/decentralized/messages.ts

@@ -1,7 +1,7 @@
-import { weights } from '../../serialization'
-import { type SignalData } from './peer'
-import { isNodeID, type NodeID } from '../types'
-import { type, type ClientConnected, type AssignNodeID, hasMessageType } from '../messages'
+import { weights } from '../../serialization/index.js'
+import { type SignalData } from './peer.js'
+import { isNodeID, type NodeID } from '../types.js'
+import { type, type ClientConnected, type AssignNodeID, hasMessageType } from '../messages.js'
 
 /// Phase 0 communication (between server and peers)
 

discojs/discojs-core/src/client/decentralized/peer.spec.ts

@@ -1,15 +1,15 @@
 import { assert } from 'chai'
 import { List, Range, Set } from 'immutable'
 
-import { Peer } from './peer'
+import { Peer } from './peer.js'
 
 describe('peer', function () {
   let peer1: Peer
   let peer2: Peer
 
   beforeEach(async () => {
-    peer1 = new Peer('1')
-    peer2 = new Peer('2', true)
+    peer1 = await Peer.init('1')
+    peer2 = await Peer.init('2', true)
     const peers = Set.of(peer1, peer2)
 
     peer1.on('signal', (signal) => { peer2.signal(signal) })

discojs/discojs-core/src/client/decentralized/peer.ts

@@ -1,9 +1,8 @@
-import { type NodeID } from '../types'
-
 import { List, Map, Range, Seq } from 'immutable'
-import wrtc from 'isomorphic-wrtc'
 import SimplePeer from 'simple-peer'
 
+import type { NodeID } from '../types.js'
+
 type MessageID = number
 type ChunkID = number
 
@@ -19,7 +18,9 @@ const TICK = 10
 // we can't use the definition in DOM as we're platform independent
 export type SignalData =
   | { type: 'answer' | 'offer' | 'pranswer' | 'rollback', sdp?: string }
-  | { type: 'transceiverRequest' | 'renegotiate' | 'candidate' }
+  | { type: 'transceiverRequest', transceiverRequest: { kind: string } }
+  | { type: 'renegotiate', renegotiate: true }
+  | { type: 'candidate', candidate: RTCIceCandidate }
 
 interface Events {
   'close': () => void
@@ -42,8 +43,6 @@ interface Events {
 //
 // see feross/simple-peer#393 for more info
 export class Peer {
-  public readonly id: NodeID
-  private readonly peer: SimplePeer.Instance
   private bufferSize?: number
 
   private sendCounter: MessageID = 0
@@ -54,9 +53,16 @@ export class Peer {
     chunks: Map<ChunkID, Buffer>
   }>()
 
-  constructor (id: NodeID, initiator: boolean = false) {
-    this.id = id
-    this.peer = new SimplePeer({ wrtc, initiator })
+  private constructor (
+    public readonly id: NodeID,
+    private readonly peer: SimplePeer.Instance
+  ) {}
+
+  static async init (id: NodeID, initiator: boolean = false): Promise<Peer> {
+    return new Peer(
+      id,
+      new SimplePeer({ wrtc: (await import('isomorphic-wrtc')).default, initiator })
+    )
   }
 
   send (msg: Buffer): void {

discojs/discojs-core/src/client/decentralized/peer_pool.spec.ts

@@ -1,11 +1,11 @@
 import { assert } from 'chai'
 import { Map, Range } from 'immutable'
-import { type messages } from '.'
-import { type } from '../messages'
-import { type PeerConnection, type EventConnection } from '../event_connection'
+import type { messages } from './index.js'
+import { type } from '../messages.js'
+import type { PeerConnection, EventConnection } from '../event_connection.js'
 
-import { PeerPool } from './peer_pool'
-import { type NodeID } from '../types'
+import { PeerPool } from './peer_pool.js'
+import type { NodeID } from '../types.js'
 
 describe('peer pool', function () {
   this.timeout(10_000)

discojs/discojs-core/src/client/decentralized/peer_pool.ts

@@ -1,8 +1,8 @@
 import { Map, type Set } from 'immutable'
 
-import { Peer, type SignalData } from './peer'
-import { type NodeID } from '../types'
-import { PeerConnection, type EventConnection } from '../event_connection'
+import { Peer, type SignalData } from './peer.js'
+import { type NodeID } from '../types.js'
+import { PeerConnection, type EventConnection } from '../event_connection.js'
 
 // TODO cleanup old peers
 
@@ -43,9 +43,12 @@ export class PeerPool {
 
     console.info(`[${this.id}] is connecting peers:`, peersToConnect.toJS())
 
-    const newPeers = Map(peersToConnect
-      .filter((id) => !this.peers.has(id))
-      .map((id) => [id, new Peer(id, id < this.id)]))
+    const newPeers = Map(await Promise.all(
+      peersToConnect
+        .filter((id) => !this.peers.has(id))
+        .map(async (id) => [id, await Peer.init(id, id < this.id)] as [string, Peer])
+        .toArray()
+    ))
 
     console.info(`[${this.id}] asked to connect new peers:`, newPeers.keySeq().toJS())
     const newPeersConnections = newPeers.map((peer, id) => new PeerConnection(this.id, peer, signallingServer))

discojs/discojs-core/src/client/event_connection.ts

@@ -1,12 +1,12 @@
 import isomorphic from 'isomorphic-ws'
-import type { Peer, SignalData } from './decentralized/peer'
-import { type NodeID } from './types'
+import type { Peer, SignalData } from './decentralized/peer.js'
+import { type NodeID } from './types.js'
 import msgpack from 'msgpack-lite'
-import * as decentralizedMessages from './decentralized/messages'
-import { type, type NarrowMessage, type Message } from './messages'
-import { timeout } from './utils'
+import * as decentralizedMessages from './decentralized/messages.js'
+import { type, type NarrowMessage, type Message } from './messages.js'
+import { timeout } from './utils.js'
 
-import { EventEmitter } from '../utils/event_emitter'
+import { EventEmitter } from '../utils/event_emitter.js'
 
 export interface EventConnection {
   on: <K extends type>(type: K, handler: (event: NarrowMessage<K>) => void) => void