From f49165d008a2e48fc9a723c3bb63e63a97e195b4 Mon Sep 17 00:00:00 2001 From: Nicholas Molnar Date: Mon, 21 Aug 2023 14:09:42 -0700 Subject: [PATCH 1/7] feat: make persistence pluggable --- src/Client.ts | 15 +++++++++++++++ src/keystore/providers/helpers.ts | 12 ++++++------ src/keystore/providers/interfaces.ts | 3 +++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/Client.ts b/src/Client.ts index 08ba0a68..30decbc2 100644 --- a/src/Client.ts +++ b/src/Client.ts @@ -32,6 +32,7 @@ import { NetworkKeystoreProvider, StaticKeystoreProvider, } from './keystore/providers' +import { LocalStoragePersistence, Persistence } from './keystore/persistence' const { Compression } = proto const { b64Decode } = fetcher @@ -144,6 +145,17 @@ export type KeyStoreOptions = { * A bundle can be retried using `Client.getKeys(...)` */ privateKeyOverride?: Uint8Array + + /** + * Override the base persistence provider. + * Defaults to LocalStoragePersistence, which is fine for most implementations + */ + basePersistence: Persistence + /** + * Whether or not the persistence provider should encrypt the values. + * Only disable if you are using a secure datastore that already has encryption + */ + disablePersistenceEncryption: boolean } export type LegacyOptions = { @@ -195,10 +207,13 @@ export function defaultOptions(opts?: Partial): ClientOptions { maxContentSize: MaxContentSize, persistConversations: true, skipContactPublishing: false, + basePersistence: new LocalStoragePersistence(), + disablePersistenceEncryption: false, keystoreProviders: defaultKeystoreProviders(), apiClientFactory: (options: NetworkOptions) => createApiClientFromOptions(options), } + if (opts?.codecs) { opts.codecs = _defaultOptions.codecs.concat(opts.codecs) } diff --git a/src/keystore/providers/helpers.ts b/src/keystore/providers/helpers.ts index 92b8f3a1..3911d902 100644 --- a/src/keystore/providers/helpers.ts +++ b/src/keystore/providers/helpers.ts @@ -1,10 +1,6 @@ import { PrivateKeyBundleV2 } from './../../crypto/PrivateKeyBundle' import { PrivateKeyBundleV1 } from '../../crypto/PrivateKeyBundle' -import { - EncryptedPersistence, - LocalStoragePersistence, - PrefixedPersistence, -} from '../persistence' +import { EncryptedPersistence, PrefixedPersistence } from '../persistence' import { KeystoreProviderOptions } from './interfaces' export const buildPersistenceFromOptions = async ( @@ -16,9 +12,13 @@ export const buildPersistenceFromOptions = async ( } const address = await keys.identityKey.publicKey.walletSignatureAddress() const prefix = `xmtp/${opts.env}/${address}/` + const basePersistence = opts.basePersistence + const shouldEncrypt = !opts.disablePersistenceEncryption return new PrefixedPersistence( prefix, - new EncryptedPersistence(new LocalStoragePersistence(), keys.identityKey) + shouldEncrypt + ? new EncryptedPersistence(basePersistence, keys.identityKey) + : basePersistence ) } diff --git a/src/keystore/providers/interfaces.ts b/src/keystore/providers/interfaces.ts index 9837786a..50570aaa 100644 --- a/src/keystore/providers/interfaces.ts +++ b/src/keystore/providers/interfaces.ts @@ -2,11 +2,14 @@ import type { XmtpEnv, PreEventCallbackOptions } from '../../Client' import type { Signer } from '../../types/Signer' import type { Keystore } from '../interfaces' import type { IApiClient } from '../../ApiClient' +import { Persistence } from '../persistence' export type KeystoreProviderOptions = { env: XmtpEnv persistConversations: boolean privateKeyOverride?: Uint8Array + basePersistence: Persistence + disablePersistenceEncryption: boolean } & PreEventCallbackOptions /** From d1ff3ebd7713f7af47675d07e1af175a79a69fb8 Mon Sep 17 00:00:00 2001 From: Nicholas Molnar Date: Mon, 21 Aug 2023 18:40:08 -0700 Subject: [PATCH 2/7] test: add base persistence --- test/keystore/providers/helpers.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/keystore/providers/helpers.ts b/test/keystore/providers/helpers.ts index 3e6d88a1..39481281 100644 --- a/test/keystore/providers/helpers.ts +++ b/test/keystore/providers/helpers.ts @@ -1,8 +1,13 @@ +import { LocalStoragePersistence } from '../../../src' + export const testProviderOptions = ( privateKeyOverride = undefined, - persistConversations = false + persistConversations = false, + basePersistence = new LocalStoragePersistence() ) => ({ env: 'local' as const, persistConversations, privateKeyOverride, + basePersistence, + disablePersistenceEncryption: false, }) From f49c83915fc8d02b99d3dfd79bcef4906730d13d Mon Sep 17 00:00:00 2001 From: Nicholas Molnar Date: Mon, 21 Aug 2023 18:52:53 -0700 Subject: [PATCH 3/7] test: handle new provider options --- .../KeyGeneratorKeystoreProvider.test.ts | 8 ++--- .../providers/NetworkKeyManager.test.ts | 5 +-- .../providers/NetworkKeystoreProvider.test.ts | 8 ++--- .../providers/StaticKeystoreProvider.test.ts | 35 +++++++++++-------- test/keystore/providers/helpers.ts | 10 +++--- 5 files changed, 38 insertions(+), 28 deletions(-) diff --git a/test/keystore/providers/KeyGeneratorKeystoreProvider.test.ts b/test/keystore/providers/KeyGeneratorKeystoreProvider.test.ts index 50f0716c..bf9c409a 100644 --- a/test/keystore/providers/KeyGeneratorKeystoreProvider.test.ts +++ b/test/keystore/providers/KeyGeneratorKeystoreProvider.test.ts @@ -18,7 +18,7 @@ describe('KeyGeneratorKeystoreProvider', () => { it('creates a key when wallet supplied', async () => { const provider = new KeyGeneratorKeystoreProvider() const keystore = await provider.newKeystore( - testProviderOptions(), + testProviderOptions({}), apiClient, wallet ) @@ -28,7 +28,7 @@ describe('KeyGeneratorKeystoreProvider', () => { it('throws KeystoreProviderUnavailableError when no wallet supplied', async () => { const provider = new KeyGeneratorKeystoreProvider() const prom = provider.newKeystore( - testProviderOptions(), + testProviderOptions({}), apiClient, undefined ) @@ -39,7 +39,7 @@ describe('KeyGeneratorKeystoreProvider', () => { const provider = new KeyGeneratorKeystoreProvider() const preCreateIdentityCallback = jest.fn() const keystore = await provider.newKeystore( - { ...testProviderOptions(), preCreateIdentityCallback }, + { ...testProviderOptions({}), preCreateIdentityCallback }, apiClient, wallet ) @@ -51,7 +51,7 @@ describe('KeyGeneratorKeystoreProvider', () => { const provider = new KeyGeneratorKeystoreProvider() const preEnableIdentityCallback = jest.fn() const keystore = await provider.newKeystore( - { ...testProviderOptions(), preEnableIdentityCallback }, + { ...testProviderOptions({}), preEnableIdentityCallback }, apiClient, wallet ) diff --git a/test/keystore/providers/NetworkKeyManager.test.ts b/test/keystore/providers/NetworkKeyManager.test.ts index 6ab6ebcf..5ef0d414 100644 --- a/test/keystore/providers/NetworkKeyManager.test.ts +++ b/test/keystore/providers/NetworkKeyManager.test.ts @@ -6,6 +6,7 @@ import { buildPersistenceFromOptions } from '../../../src/keystore/providers/hel import NetworkKeyManager from '../../../src/keystore/providers/NetworkKeyManager' import { Signer } from '../../../src/types/Signer' import { newWallet, pollFor, sleep, wrapAsLedgerWallet } from '../../helpers' +import { testProviderOptions } from './helpers' describe('NetworkKeyManager', () => { let wallet: Signer @@ -100,13 +101,13 @@ describe('NetworkKeyManager', () => { it('respects the options provided', async () => { const bundle = await PrivateKeyBundleV1.generate(wallet) const shouldBeUndefined = await buildPersistenceFromOptions( - { persistConversations: false, env: 'local' }, + testProviderOptions({ persistConversations: false }), bundle ) expect(shouldBeUndefined).toBeUndefined() const shouldBeDefined = await buildPersistenceFromOptions( - { persistConversations: true, env: 'local' }, + testProviderOptions({ persistConversations: true }), bundle ) expect(shouldBeDefined).toBeInstanceOf(PrefixedPersistence) diff --git a/test/keystore/providers/NetworkKeystoreProvider.test.ts b/test/keystore/providers/NetworkKeystoreProvider.test.ts index e3e0a33a..51490dca 100644 --- a/test/keystore/providers/NetworkKeystoreProvider.test.ts +++ b/test/keystore/providers/NetworkKeystoreProvider.test.ts @@ -28,7 +28,7 @@ describe('NetworkKeystoreProvider', () => { it('fails gracefully when no keys are found', async () => { const provider = new NetworkKeystoreProvider() expect( - provider.newKeystore(testProviderOptions(), apiClient, wallet) + provider.newKeystore(testProviderOptions({}), apiClient, wallet) ).rejects.toThrow(KeystoreProviderUnavailableError) }) @@ -41,7 +41,7 @@ describe('NetworkKeystoreProvider', () => { const provider = new NetworkKeystoreProvider() const keystore = await provider.newKeystore( - testProviderOptions(), + testProviderOptions({}), apiClient, wallet ) @@ -74,7 +74,7 @@ describe('NetworkKeystoreProvider', () => { // Now try and load it const provider = new NetworkKeystoreProvider() const keystore = await provider.newKeystore( - testProviderOptions(), + testProviderOptions({}), apiClient, wallet ) @@ -91,7 +91,7 @@ describe('NetworkKeystoreProvider', () => { const provider = new NetworkKeystoreProvider() const mockNotifier = jest.fn() await provider.newKeystore( - { ...testProviderOptions(), preEnableIdentityCallback: mockNotifier }, + { ...testProviderOptions({}), preEnableIdentityCallback: mockNotifier }, apiClient, wallet ) diff --git a/test/keystore/providers/StaticKeystoreProvider.test.ts b/test/keystore/providers/StaticKeystoreProvider.test.ts index 880ec415..c46b0628 100644 --- a/test/keystore/providers/StaticKeystoreProvider.test.ts +++ b/test/keystore/providers/StaticKeystoreProvider.test.ts @@ -1,6 +1,7 @@ import { privateKey } from '@xmtp/proto' import { PrivateKeyBundleV1 } from '../../../src/crypto' import { newWallet } from '../../helpers' +import { testProviderOptions } from './helpers' import StaticKeystoreProvider from '../../../src/keystore/providers/StaticKeystoreProvider' import { KeystoreProviderUnavailableError } from '../../../src/keystore/providers/errors' @@ -14,31 +15,37 @@ describe('StaticKeystoreProvider', () => { v2: undefined, }).finish() const provider = new StaticKeystoreProvider() - const keystore = await provider.newKeystore({ - privateKeyOverride: keyBytes, - env: ENV, - persistConversations: false, - }) + const keystore = await provider.newKeystore( + testProviderOptions({ + privateKeyOverride: keyBytes, + env: ENV, + persistConversations: false, + }) + ) expect(keystore).not.toBeNull() }) it('throws with an unset key', async () => { expect( - new StaticKeystoreProvider().newKeystore({ - env: ENV, - persistConversations: false, - }) + new StaticKeystoreProvider().newKeystore( + testProviderOptions({ + env: ENV, + persistConversations: false, + }) + ) ).rejects.toThrow(KeystoreProviderUnavailableError) }) it('fails with an invalid key', async () => { expect( - new StaticKeystoreProvider().newKeystore({ - privateKeyOverride: Uint8Array.from([1, 2, 3]), - env: ENV, - persistConversations: false, - }) + new StaticKeystoreProvider().newKeystore( + testProviderOptions({ + privateKeyOverride: Uint8Array.from([1, 2, 3]), + env: ENV, + persistConversations: false, + }) + ) ).rejects.toThrow() }) }) diff --git a/test/keystore/providers/helpers.ts b/test/keystore/providers/helpers.ts index 39481281..5b0ac514 100644 --- a/test/keystore/providers/helpers.ts +++ b/test/keystore/providers/helpers.ts @@ -1,11 +1,13 @@ import { LocalStoragePersistence } from '../../../src' +import { KeystoreProviderOptions } from '../../../src/keystore/providers' -export const testProviderOptions = ( +export const testProviderOptions = ({ privateKeyOverride = undefined, persistConversations = false, - basePersistence = new LocalStoragePersistence() -) => ({ - env: 'local' as const, + basePersistence = new LocalStoragePersistence(), + env = 'local' as const, +}: Partial) => ({ + env, persistConversations, privateKeyOverride, basePersistence, From d1d49b1c3d82ba5c829d1dfd892db85cffe02cd8 Mon Sep 17 00:00:00 2001 From: Nicholas Molnar Date: Tue, 22 Aug 2023 11:06:25 -0700 Subject: [PATCH 4/7] build: add retry fn --- src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index cc9b66ec..6aea3faf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,3 @@ -import { Authenticator } from './authn/interfaces' export { Message, DecodedMessage, @@ -73,10 +72,11 @@ export { UnsubscribeFn, OnConnectionLostCallback, } from './ApiClient' -export { Authenticator, AuthCache } from './authn' +export { Authenticator, AuthCache, LocalAuthenticator } from './authn' export { nsToDate, dateToNs, + retry, fromNanoString, toNanoString, mapPaginatedStream, From 413e02bd2e7ba62ca6d42f69496747d650779aa1 Mon Sep 17 00:00:00 2001 From: Nicholas Molnar Date: Wed, 23 Aug 2023 08:57:03 -0700 Subject: [PATCH 5/7] test: add a test to ensure persistence is used --- src/Client.ts | 18 +++++++++++++++--- test/Client.test.ts | 23 ++++++++++++++++++++++- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/Client.ts b/src/Client.ts index cfb718de..12fc15e3 100644 --- a/src/Client.ts +++ b/src/Client.ts @@ -32,7 +32,11 @@ import { NetworkKeystoreProvider, StaticKeystoreProvider, } from './keystore/providers' -import { LocalStoragePersistence, Persistence } from './keystore/persistence' +import { + LocalStoragePersistence, + Persistence, + PrefixedPersistence, +} from './keystore/persistence' const { Compression } = proto // eslint-disable @typescript-eslint/explicit-module-boundary-types @@ -229,6 +233,7 @@ export default class Client { apiClient: ApiClient contacts: Set // address which we have connected to publicKeyBundle: PublicKeyBundle + persistence: Persistence private knownPublicKeyBundles: Map< string, PublicKeyBundle | SignedPublicKeyBundle @@ -244,7 +249,8 @@ export default class Client { publicKeyBundle: PublicKeyBundle, apiClient: ApiClient, backupClient: BackupClient, - keystore: Keystore + keystore: Keystore, + persistence: Persistence ) { this.contacts = new Set() this.knownPublicKeyBundles = new Map< @@ -255,6 +261,7 @@ export default class Client { this.keystore = keystore this.publicKeyBundle = publicKeyBundle this.address = publicKeyBundle.walletSignatureAddress() + this.persistence = persistence this._conversations = new Conversations(this) this._codecs = new Map() this._maxContentSize = MaxContentSize @@ -296,11 +303,16 @@ export default class Client { const address = publicKeyBundle.walletSignatureAddress() apiClient.setAuthenticator(new KeystoreAuthenticator(keystore)) const backupClient = await Client.setupBackupClient(address, options.env) + const persistence = new PrefixedPersistence( + `xmtp/${options.env}/${address}/`, + options.basePersistence + ) const client = new Client( publicKeyBundle, apiClient, backupClient, - keystore + keystore, + persistence ) await client.init(options) return client diff --git a/test/Client.test.ts b/test/Client.test.ts index c326a0d4..09d199c4 100644 --- a/test/Client.test.ts +++ b/test/Client.test.ts @@ -8,7 +8,13 @@ import { } from './helpers' import { buildUserContactTopic } from '../src/utils' import Client, { ClientOptions } from '../src/Client' -import { ApiUrls, Compression, HttpApiClient, PublishParams } from '../src' +import { + ApiUrls, + Compression, + HttpApiClient, + LocalStoragePersistence, + PublishParams, +} from '../src' import NetworkKeyManager from '../src/keystore/providers/NetworkKeyManager' import TopicPersistence from '../src/keystore/persistence/TopicPersistence' import { PrivateKeyBundleV1 } from '../src/crypto' @@ -336,4 +342,19 @@ describe('ClientOptions', () => { expect(c).rejects.toThrow(expectedError) }) }) + + describe('pluggable persistence', () => { + it('allows for an override of the persistence engine', async () => { + class MyNewPersistence extends LocalStoragePersistence { + async getItem(key: string): Promise { + throw new Error('MyNewPersistence') + } + } + + const c = newLocalHostClient({ + basePersistence: new MyNewPersistence(), + }) + expect(c).rejects.toThrow('MyNewPersistence') + }) + }) }) From de94c3d3df879371d45ae3000b5e95612304efce Mon Sep 17 00:00:00 2001 From: Nicholas Molnar Date: Wed, 23 Aug 2023 09:37:38 -0700 Subject: [PATCH 6/7] chore: remove persistence from client --- src/Client.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/Client.ts b/src/Client.ts index 12fc15e3..2b741728 100644 --- a/src/Client.ts +++ b/src/Client.ts @@ -233,7 +233,6 @@ export default class Client { apiClient: ApiClient contacts: Set // address which we have connected to publicKeyBundle: PublicKeyBundle - persistence: Persistence private knownPublicKeyBundles: Map< string, PublicKeyBundle | SignedPublicKeyBundle @@ -249,8 +248,7 @@ export default class Client { publicKeyBundle: PublicKeyBundle, apiClient: ApiClient, backupClient: BackupClient, - keystore: Keystore, - persistence: Persistence + keystore: Keystore ) { this.contacts = new Set() this.knownPublicKeyBundles = new Map< @@ -261,7 +259,6 @@ export default class Client { this.keystore = keystore this.publicKeyBundle = publicKeyBundle this.address = publicKeyBundle.walletSignatureAddress() - this.persistence = persistence this._conversations = new Conversations(this) this._codecs = new Map() this._maxContentSize = MaxContentSize @@ -303,16 +300,11 @@ export default class Client { const address = publicKeyBundle.walletSignatureAddress() apiClient.setAuthenticator(new KeystoreAuthenticator(keystore)) const backupClient = await Client.setupBackupClient(address, options.env) - const persistence = new PrefixedPersistence( - `xmtp/${options.env}/${address}/`, - options.basePersistence - ) const client = new Client( publicKeyBundle, apiClient, backupClient, - keystore, - persistence + keystore ) await client.init(options) return client From 3682e98cb2484d3d29b3010f10d032698339d7cb Mon Sep 17 00:00:00 2001 From: Nicholas Molnar Date: Wed, 23 Aug 2023 13:48:32 -0700 Subject: [PATCH 7/7] chore: remove unused import --- src/Client.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Client.ts b/src/Client.ts index 2b741728..cfb718de 100644 --- a/src/Client.ts +++ b/src/Client.ts @@ -32,11 +32,7 @@ import { NetworkKeystoreProvider, StaticKeystoreProvider, } from './keystore/providers' -import { - LocalStoragePersistence, - Persistence, - PrefixedPersistence, -} from './keystore/persistence' +import { LocalStoragePersistence, Persistence } from './keystore/persistence' const { Compression } = proto // eslint-disable @typescript-eslint/explicit-module-boundary-types