Merge branch 'master' into dependabot/npm_and_yarn/ava-6.1.1

Author: sosnovsky

.github/codeql/codeql-config.yml

@@ -2,4 +2,4 @@ paths-ignore:
   - node_modules
   - extension/lib
   - extension/types
-  - test/source/**/*.ts
+  - test/source/

.semaphore/semaphore.yml

@@ -1,5 +1,5 @@
 version: v1.0
-name: Flowcrypt Node Core Tests
+name: FlowCrypt Extension Tests
 agent:
   machine:
     type: e2-standard-4

extension/js/common/api/account-servers/external-service.ts

@@ -13,7 +13,6 @@ import { Buf } from '../../core/buf.js';
 import { ClientConfigurationError, ClientConfigurationJson } from '../../client-configuration.js';
 import { InMemoryStore } from '../../platform/store/in-memory-store.js';
 import { Serializable } from '../../platform/store/abstract-store.js';
-import { GoogleOAuth } from '../authentication/google/google-oauth.js';
 import { AuthenticationConfiguration } from '../../authentication-configuration.js';
 import { Xss } from '../../platform/xss.js';
 
@@ -193,27 +192,6 @@ export class ExternalService extends Api {
           method: 'POST',
         }
       : undefined;
-    try {
-      return await ExternalService.apiCall(this.url, path, values, progress, await this.authHdr(), 'json');
-    } catch (firstAttemptErr) {
-      const idToken = await InMemoryStore.get(this.acctEmail, InMemoryStoreKeys.ID_TOKEN);
-      if (ApiErr.isAuthErr(firstAttemptErr) && idToken) {
-        // force refresh token
-        const { email } = GoogleOAuth.parseIdToken(idToken);
-        if (email) {
-          return await ExternalService.apiCall(
-            this.url,
-            path,
-            values,
-            progress,
-            {
-              authorization: await GoogleOAuth.googleApiAuthHeader(email, true),
-            },
-            'json'
-          );
-        }
-      }
-      throw firstAttemptErr;
-    }
+    return await ExternalService.apiCall(this.url, path, values, progress, await this.authHdr(), 'json');
   };
 }

extension/js/common/api/shared/api.ts

@@ -412,7 +412,24 @@ export class Api {
       const result = await Api.ajaxWithJquery(req, resFmt, formattedData);
       return result as FetchResult<T, RT>;
     } else {
-      return await Api.ajax(req, resFmt);
+      try {
+        return await Api.ajax(req, resFmt);
+      } catch (firstAttemptErr) {
+        const idToken = headers?.authorization?.split('Bearer ')?.[1];
+        if (ApiErr.isAuthErr(firstAttemptErr) && idToken) {
+          // Needed authorization from the service worker side to avoid circular dependency injection errors
+          // that occur when importing GoogleAuth directly.
+          const authorization = await BrowserMsg.send.bg.await.getGoogleApiAuthorization({ idToken });
+          if (authorization) {
+            const updatedReq = {
+              ...req,
+              headers: { authorization },
+            };
+            return await Api.ajax(updatedReq, resFmt);
+          }
+        }
+        throw firstAttemptErr;
+      }
     }
   }
 

extension/js/common/browser/browser-msg.ts

@@ -77,6 +77,7 @@ export namespace Bm {
     expiration: number | undefined;
   };
   export type InMemoryStoreGet = { acctEmail: string; key: string };
+  export type GetGoogleApiAuthorization = { idToken: string };
   export type ReconnectAcctAuthPopup = { acctEmail: string; scopes?: string[]; screenDimensions: ScreenDimensions };
   export type Ajax = { req: ApiAjax; resFmt: ResFmt };
   export type AjaxProgress = { operationId: string; percent?: number; loaded: number; total: number; expectedTransferSize: number };
@@ -99,6 +100,7 @@ export namespace Bm {
     export type InMemoryStoreGet = string | undefined;
     // eslint-disable-next-line @typescript-eslint/no-invalid-void-type
     export type InMemoryStoreSet = void;
+    export type GetGoogleApiAuthorization = string | undefined;
     export type ReconnectAcctAuthPopup = AuthRes;
     export type AjaxGmailAttachmentGetChunk = { chunk: Buf };
     export type ExpirationCacheGet<V> = Promise<V | undefined>;
@@ -113,6 +115,7 @@ export namespace Bm {
       | GetActiveTabInfo
       | ReconnectAcctAuthPopup
       | InMemoryStoreGet
+      | GetGoogleApiAuthorization
       | InMemoryStoreSet
       | ExpirationCacheGet<unknown>
       | ExpirationCacheSet
@@ -124,6 +127,7 @@ export namespace Bm {
   export type AnyRequest =
     | PassphraseEntry
     | OpenPage
+    | GetGoogleApiAuthorization
     | ExpirationCacheGet
     | ExpirationCacheSet<unknown>
     | ExpirationCacheDeleteExpired
@@ -202,6 +206,8 @@ export class BrowserMsg {
         getActiveTabInfo: () => BrowserMsg.sendAwait(undefined, 'get_active_tab_info', undefined, true) as Promise<Bm.Res.GetActiveTabInfo>,
         inMemoryStoreGet: (bm: Bm.InMemoryStoreGet) => BrowserMsg.sendAwait(undefined, 'inMemoryStoreGet', bm, true) as Promise<Bm.Res.InMemoryStoreGet>,
         inMemoryStoreSet: (bm: Bm.InMemoryStoreSet) => BrowserMsg.sendAwait(undefined, 'inMemoryStoreSet', bm, true) as Promise<Bm.Res.InMemoryStoreSet>,
+        getGoogleApiAuthorization: (bm: Bm.GetGoogleApiAuthorization) =>
+          BrowserMsg.sendAwait(undefined, 'getGoogleApiAuthorization', bm, true) as Promise<Bm.Res.GetGoogleApiAuthorization>,
         db: (bm: Bm.Db): Promise<Bm.Res.Db> => BrowserMsg.sendAwait(undefined, 'db', bm, true) as Promise<Bm.Res.Db>,
         ajax: (bm: Bm.Ajax): Promise<Bm.Res.Ajax> => BrowserMsg.sendAwait(undefined, 'ajax', bm, true) as Promise<Bm.Res.Ajax>,
         ajaxGmailAttachmentGetChunk: (bm: Bm.AjaxGmailAttachmentGetChunk) =>

extension/js/common/platform/catch.ts

@@ -281,6 +281,14 @@ export class Catch {
     }
   }
 
+  private static groupSimilarReports(value: string): string {
+    return value
+      .replace(/chrome-extension:\/\/[^\/]+\//, 'chrome-extension://EXTENSION_ID/')
+      .replace(/https:\/\/www\.googleapis\.com\/gmail\/v1\/users\/me\/threads\/[^\/]+/, 'https://www.googleapis.com/gmail/v1/users/me/threads/THREAD_ID')
+      .replace(/https:\/\/www\.googleapis\.com\/gmail\/v1\/users\/me\/messages\/[^\/]+/, 'https://www.googleapis.com/gmail/v1/users/me/messages/MESSAGE_ID')
+      .replace(/https:\/\/www\.googleapis\.com\/gmail\/v1\/users\/me\/drafts\/[^\/]+/, 'https://www.googleapis.com/gmail/v1/users/me/drafts/DRAFT_ID');
+  }
+
   private static formatExceptionForReport(thrown: unknown, line?: number, col?: number): ErrorReport {
     if (!line || !col) {
       const { line: parsedLine, col: parsedCol } = Catch.getErrorLineAndCol(thrown);
@@ -298,11 +306,11 @@ export class Catch {
     const exception = Catch.formExceptionFromThrown(thrown);
     return {
       name: exception.name.substring(0, 50),
-      message: exception.message.substring(0, 200),
-      url: location.href.split('?')[0],
+      message: Catch.groupSimilarReports(exception.message.substring(0, 200)),
+      url: Catch.groupSimilarReports(location.href.split('?')[0]),
       line: line || 1,
       col: col || 1,
-      trace: exception.stack || '',
+      trace: Catch.groupSimilarReports(exception.stack || ''),
       version: VERSION,
       environment: Catch.RUNTIME_ENVIRONMENT,
       product: 'web-ext',

extension/js/content_scripts/webmail/setup-webmail-content-script.ts renamed to extension/js/content_scripts/webmail/generic/setup-webmail-content-script.ts

@@ -3,36 +3,37 @@
 'use strict';
 
 import Swal from 'sweetalert2';
-import { AccountServer } from '../../common/api/account-server.js';
-import { KeyManager } from '../../common/api/key-server/key-manager.js';
-import { ApiErr, BackendAuthErr } from '../../common/api/shared/api-error.js';
-import { BrowserMsgCommonHandlers } from '../../common/browser/browser-msg-common-handlers.js';
-import { Bm, BrowserMsg, TabIdRequiredError } from '../../common/browser/browser-msg.js';
-import { ContentScriptWindow } from '../../common/browser/browser-window.js';
-import { Env, WebMailName } from '../../common/browser/env.js';
-import { Time } from '../../common/browser/time.js';
-import { CommonHandlers, Ui } from '../../common/browser/ui.js';
-import { ClientConfiguration, ClientConfigurationError } from '../../common/client-configuration.js';
-import { Str, Url } from '../../common/core/common.js';
-import { InMemoryStoreKeys, VERSION } from '../../common/core/const.js';
-import { getLocalKeyExpiration, processAndStoreKeysFromEkmLocally } from '../../common/helpers.js';
-import { Injector } from '../../common/inject.js';
-import { Lang } from '../../common/lang.js';
-import { Notifications } from '../../common/notifications.js';
-import { Catch } from '../../common/platform/catch.js';
-import { AcctStore } from '../../common/platform/store/acct-store.js';
-import { GlobalStore } from '../../common/platform/store/global-store.js';
-import { InMemoryStore } from '../../common/platform/store/in-memory-store.js';
-import { WebmailVariantString, XssSafeFactory } from '../../common/xss-safe-factory.js';
-import { RelayManager } from '../../common/relay-manager.js';
+import { AccountServer } from '../../../common/api/account-server.js';
+import { KeyManager } from '../../../common/api/key-server/key-manager.js';
+import { ApiErr, BackendAuthErr } from '../../../common/api/shared/api-error.js';
+import { BrowserMsgCommonHandlers } from '../../../common/browser/browser-msg-common-handlers.js';
+import { Bm, BrowserMsg, TabIdRequiredError } from '../../../common/browser/browser-msg.js';
+import { ContentScriptWindow } from '../../../common/browser/browser-window.js';
+import { Env, WebMailName } from '../../../common/browser/env.js';
+import { Time } from '../../../common/browser/time.js';
+import { CommonHandlers, Ui } from '../../../common/browser/ui.js';
+import { ClientConfiguration, ClientConfigurationError } from '../../../common/client-configuration.js';
+import { Str, Url } from '../../../common/core/common.js';
+import { InMemoryStoreKeys, VERSION } from '../../../common/core/const.js';
+import { getLocalKeyExpiration, processAndStoreKeysFromEkmLocally } from '../../../common/helpers.js';
+import { Injector } from '../../../common/inject.js';
+import { Lang } from '../../../common/lang.js';
+import { Notifications } from '../../../common/notifications.js';
+import { Catch } from '../../../common/platform/catch.js';
+import { AcctStore } from '../../../common/platform/store/acct-store.js';
+import { GlobalStore } from '../../../common/platform/store/global-store.js';
+import { InMemoryStore } from '../../../common/platform/store/in-memory-store.js';
+import { WebmailVariantString, XssSafeFactory } from '../../../common/xss-safe-factory.js';
+import { RelayManager } from '../../../common/relay-manager.js';
+import { WebmailElementReplacer } from './webmail-element-replacer.js';
 
 export type WebmailVariantObject = {
   newDataLayer: undefined | boolean;
   newUi: undefined | boolean;
   email: undefined | string;
   gmailVariant: WebmailVariantString;
 };
-export type IntervalFunction = { interval: number; handler: () => void };
+
 type WebmailSpecificInfo = {
   name: WebMailName;
   variant: WebmailVariantString;
@@ -45,17 +46,9 @@ type WebmailSpecificInfo = {
     inject: Injector,
     notifications: Notifications,
     factory: XssSafeFactory,
-    notifyMurdered: () => void,
     relayManager: RelayManager
   ) => Promise<void>;
 };
-export interface WebmailElementReplacer {
-  getIntervalFunctions: () => IntervalFunction[];
-  setReplyBoxEditable: () => Promise<void>;
-  reinsertReplyBox: (replyMsgId: string) => void;
-  scrollToReplyBox: (replyMsgId: string) => void;
-  scrollToCursorInReplyBox: (replyMsgId: string, cursorOffsetTop: number) => void;
-}
 
 const win = window as unknown as ContentScriptWindow;
 
@@ -275,23 +268,6 @@ export const contentScriptSetupIfVacant = async (webmailSpecific: WebmailSpecifi
     }
   };
 
-  const notifyMurdered = () => {
-    const notifEl = document.getElementsByClassName('webmail_notifications')[0];
-    const div = document.createElement('div');
-    div.innerText = 'FlowCrypt has updated, please reload the tab. ';
-    div.classList.add('webmail_notification');
-    const a = document.createElement('a');
-    a.href = '#';
-    a.onclick = function () {
-      const parent = (this as HTMLAnchorElement).parentNode as HTMLElement | undefined;
-      parent?.remove();
-    };
-    a.textContent = 'close';
-    div.appendChild(a);
-    notifEl.textContent = '';
-    notifEl.appendChild(div);
-  };
-
   const showPassphraseDialog = async (factory: XssSafeFactory, { longids, type, initiatorFrameId }: Bm.PassphraseDialog) => {
     await factory.showPassphraseDialog(longids, type, initiatorFrameId);
   };
@@ -451,7 +427,7 @@ export const contentScriptSetupIfVacant = async (webmailSpecific: WebmailSpecifi
         ppEvent,
         Catch.try(() => notifyExpiringKeys(acctEmail, clientConfiguration, notifications))
       );
-      await webmailSpecific.start(acctEmail, clientConfiguration, inject, notifications, factory, notifyMurdered, relayManager);
+      await webmailSpecific.start(acctEmail, clientConfiguration, inject, notifications, factory, relayManager);
     } catch (e) {
       if (e instanceof TabIdRequiredError) {
         console.error(`FlowCrypt cannot start: ${String(e)}`);
@@ -527,3 +503,28 @@ export const contentScriptSetupIfVacant = async (webmailSpecific: WebmailSpecifi
     }
   }
 };
+
+/**
+ * This happens when Firefox (or possibly Thunderbird) just updated FlowCrypt.
+ *
+ * Previous (meaning this currently running) instance of FlowCrypt will no longer
+ *   have access to its various classes or global variables, and is left in a
+ *   semi-functioning state. The best we can do is to ask the user to reload
+ *   the tab, which will load the newly updated version of the extension cleanly.
+ */
+export const notifyMurdered = () => {
+  const notifEl = document.getElementsByClassName('webmail_notifications')[0];
+  const div = document.createElement('div');
+  div.innerText = 'FlowCrypt has updated, please reload the tab. ';
+  div.classList.add('webmail_notification');
+  const a = document.createElement('a');
+  a.href = '#';
+  a.onclick = function () {
+    const parent = (this as HTMLAnchorElement).parentNode as HTMLElement | undefined;
+    parent?.remove();
+  };
+  a.textContent = 'close';
+  div.appendChild(a);
+  notifEl.textContent = '';
+  notifEl.appendChild(div);
+};

extension/js/content_scripts/webmail/generic/webmail-element-replacer.ts

@@ -0,0 +1,33 @@
+/* ©️ 2016 - present FlowCrypt a.s. Limitations apply. Contact [email protected] */
+
+import { ContentScriptWindow } from '../../../common/browser/browser-window';
+import { notifyMurdered } from './setup-webmail-content-script';
+
+export type IntervalFunction = { interval: number; handler: () => void };
+
+export abstract class WebmailElementReplacer {
+  private replacePgpElsInterval: number;
+
+  public abstract getIntervalFunctions: () => IntervalFunction[];
+  public abstract setReplyBoxEditable: () => Promise<void>;
+  public abstract reinsertReplyBox: (replyMsgId: string) => void;
+  public abstract scrollToReplyBox: (replyMsgId: string) => void;
+  public abstract scrollToCursorInReplyBox: (replyMsgId: string, cursorOffsetTop: number) => void;
+
+  public runIntervalFunctionsPeriodically = () => {
+    const intervalFunctions = this.getIntervalFunctions();
+    for (const intervalFunction of intervalFunctions) {
+      intervalFunction.handler();
+      this.replacePgpElsInterval = (window as unknown as ContentScriptWindow).TrySetDestroyableInterval(() => {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        if (typeof (window as any).$ === 'function') {
+          intervalFunction.handler();
+        } else {
+          // firefox will unload jquery when extension is restarted or updated
+          clearInterval(this.replacePgpElsInterval);
+          notifyMurdered();
+        }
+      }, intervalFunction.interval);
+    }
+  };
+}

extension/js/content_scripts/webmail/gmail-element-replacer.ts renamed to extension/js/content_scripts/webmail/gmail/gmail-element-replacer.ts

@@ -2,34 +2,34 @@
 
 'use strict';
 
-import { Dict, Str } from '../../common/core/common.js';
-import { FactoryReplyParams, XssSafeFactory } from '../../common/xss-safe-factory.js';
-import { IntervalFunction, WebmailElementReplacer } from './setup-webmail-content-script.js';
-import { ApiErr } from '../../common/api/shared/api-error.js';
-import { Attachment } from '../../common/core/attachment.js';
-import { BrowserMsg } from '../../common/browser/browser-msg.js';
-import { Catch } from '../../common/platform/catch.js';
-import { GlobalStore, LocalDraft } from '../../common/platform/store/global-store.js';
-import { Injector } from '../../common/inject.js';
-import { PubLookup } from '../../common/api/pub-lookup.js';
-import { Notifications } from '../../common/notifications.js';
-import { PgpArmor } from '../../common/core/crypto/pgp/pgp-armor.js';
-import { Ui } from '../../common/browser/ui.js';
-import { WebmailCommon } from '../../common/webmail.js';
-import { Xss } from '../../common/platform/xss.js';
-import { ClientConfiguration } from '../../common/client-configuration.js';
+import { Dict, Str } from '../../../common/core/common.js';
+import { FactoryReplyParams, XssSafeFactory } from '../../../common/xss-safe-factory.js';
+import { ApiErr } from '../../../common/api/shared/api-error.js';
+import { Attachment } from '../../../common/core/attachment.js';
+import { BrowserMsg } from '../../../common/browser/browser-msg.js';
+import { Catch } from '../../../common/platform/catch.js';
+import { GlobalStore, LocalDraft } from '../../../common/platform/store/global-store.js';
+import { Injector } from '../../../common/inject.js';
+import { PubLookup } from '../../../common/api/pub-lookup.js';
+import { Notifications } from '../../../common/notifications.js';
+import { PgpArmor } from '../../../common/core/crypto/pgp/pgp-armor.js';
+import { Ui } from '../../../common/browser/ui.js';
+import { WebmailCommon } from '../../../common/webmail.js';
+import { Xss } from '../../../common/platform/xss.js';
+import { ClientConfiguration } from '../../../common/client-configuration.js';
 // todo: can we somehow define a purely relay class for ContactStore to clearly show that crypto-libraries are not loaded and can't be used?
-import { ContactStore } from '../../common/platform/store/contact-store.js';
-import { MessageRenderer } from '../../common/message-renderer.js';
-import { RelayManager } from '../../common/relay-manager.js';
-import { MessageInfo } from '../../common/render-message.js';
+import { ContactStore } from '../../../common/platform/store/contact-store.js';
+import { MessageRenderer } from '../../../common/message-renderer.js';
+import { RelayManager } from '../../../common/relay-manager.js';
+import { MessageInfo } from '../../../common/render-message.js';
 import { GmailLoaderContext } from './gmail-loader-context.js';
-import { JQueryEl } from '../../common/loader-context-interface.js';
-import { MessageBody, Mime } from '../../common/core/mime.js';
-import { MsgBlock } from '../../common/core/msg-block.js';
-import { ReplyOption } from '../../../chrome/elements/compose-modules/compose-reply-btn-popover-module.js';
+import { JQueryEl } from '../../../common/loader-context-interface.js';
+import { MessageBody, Mime } from '../../../common/core/mime.js';
+import { MsgBlock } from '../../../common/core/msg-block.js';
+import { ReplyOption } from '../../../../chrome/elements/compose-modules/compose-reply-btn-popover-module.js';
+import { WebmailElementReplacer, IntervalFunction } from '../generic/webmail-element-replacer.js';
 
-export class GmailElementReplacer implements WebmailElementReplacer {
+export class GmailElementReplacer extends WebmailElementReplacer {
   private debug = false;
 
   private recipientHasPgpCache: Dict<boolean> = {};
@@ -74,6 +74,7 @@ export class GmailElementReplacer implements WebmailElementReplacer {
     private readonly notifications: Notifications,
     private readonly relayManager: RelayManager
   ) {
+    super();
     this.webmailCommon = new WebmailCommon(acctEmail, injector);
     this.pubLookup = new PubLookup(clientConfiguration);
   }