Browser MV3: Default store values to session storage (#8844)

* Introduce browser large object storage location.

This location is encrypted and serialized to disk in order to allow for storage of uncountable things like vault items that take a significant amount of time to prepare, but are not guaranteed to fit within session storage.

however, limit the need to write to disk is a big benefit, so _most_ things are written to storage.session instead, where things specifically flagged as large will be moved to disk-backed memory

* Store derived values in large object store for browser

* Fix AbstractMemoryStorageService implementation

apps/browser/src/background/main.background.ts

@@ -111,7 +111,6 @@ import { KeyGenerationService } from "@bitwarden/common/platform/services/key-ge
 import { MemoryStorageService } from "@bitwarden/common/platform/services/memory-storage.service";
 import { MigrationBuilderService } from "@bitwarden/common/platform/services/migration-builder.service";
 import { MigrationRunner } from "@bitwarden/common/platform/services/migration-runner";
-import { StorageServiceProvider } from "@bitwarden/common/platform/services/storage-service.provider";
 import { SystemService } from "@bitwarden/common/platform/services/system.service";
 import { UserKeyInitService } from "@bitwarden/common/platform/services/user-key-init.service";
 import { WebCryptoFunctionService } from "@bitwarden/common/platform/services/web-crypto-function.service";
@@ -226,6 +225,7 @@ import { BackgroundPlatformUtilsService } from "../platform/services/platform-ut
 import { BrowserPlatformUtilsService } from "../platform/services/platform-utils/browser-platform-utils.service";
 import { BackgroundDerivedStateProvider } from "../platform/state/background-derived-state.provider";
 import { BackgroundMemoryStorageService } from "../platform/storage/background-memory-storage.service";
+import { BrowserStorageServiceProvider } from "../platform/storage/browser-storage-service.provider";
 import { ForegroundMemoryStorageService } from "../platform/storage/foreground-memory-storage.service";
 import { fromChromeRuntimeMessaging } from "../platform/utils/from-chrome-runtime-messaging";
 import VaultTimeoutService from "../services/vault-timeout/vault-timeout.service";
@@ -246,6 +246,8 @@ export default class MainBackground {
   secureStorageService: AbstractStorageService;
   memoryStorageService: AbstractMemoryStorageService;
   memoryStorageForStateProviders: AbstractMemoryStorageService & ObservableStorageService;
+  largeObjectMemoryStorageForStateProviders: AbstractMemoryStorageService &
+    ObservableStorageService;
   i18nService: I18nServiceAbstraction;
   platformUtilsService: PlatformUtilsServiceAbstraction;
   logService: LogServiceAbstraction;
@@ -424,12 +426,16 @@ export default class MainBackground {
       ? mv3MemoryStorageCreator("stateService")
       : new MemoryStorageService();
     this.memoryStorageForStateProviders = BrowserApi.isManifestVersion(3)
-      ? mv3MemoryStorageCreator("stateProviders")
-      : new BackgroundMemoryStorageService();
+      ? new BrowserMemoryStorageService() // mv3 stores to storage.session
+      : new BackgroundMemoryStorageService(); // mv2 stores to memory
+    this.largeObjectMemoryStorageForStateProviders = BrowserApi.isManifestVersion(3)
+      ? mv3MemoryStorageCreator("stateProviders") // mv3 stores to local-backed session storage
+      : this.memoryStorageForStateProviders; // mv2 stores to the same location
 
-    const storageServiceProvider = new StorageServiceProvider(
+    const storageServiceProvider = new BrowserStorageServiceProvider(
       this.storageService,
       this.memoryStorageForStateProviders,
+      this.largeObjectMemoryStorageForStateProviders,
     );
 
     this.globalStateProvider = new DefaultGlobalStateProvider(storageServiceProvider);
@@ -466,9 +472,7 @@ export default class MainBackground {
       this.accountService,
       this.singleUserStateProvider,
     );
-    this.derivedStateProvider = new BackgroundDerivedStateProvider(
-      this.memoryStorageForStateProviders,
-    );
+    this.derivedStateProvider = new BackgroundDerivedStateProvider(storageServiceProvider);
     this.stateProvider = new DefaultStateProvider(
       this.activeUserStateProvider,
       this.singleUserStateProvider,

apps/browser/src/platform/background/service-factories/derived-state-provider.factory.ts

@@ -4,14 +4,14 @@ import { BackgroundDerivedStateProvider } from "../../state/background-derived-s
 
 import { CachedServices, FactoryOptions, factory } from "./factory-options";
 import {
-  MemoryStorageServiceInitOptions,
-  observableMemoryStorageServiceFactory,
-} from "./storage-service.factory";
+  StorageServiceProviderInitOptions,
+  storageServiceProviderFactory,
+} from "./storage-service-provider.factory";
 
 type DerivedStateProviderFactoryOptions = FactoryOptions;
 
 export type DerivedStateProviderInitOptions = DerivedStateProviderFactoryOptions &
-  MemoryStorageServiceInitOptions;
+  StorageServiceProviderInitOptions;
 
 export async function derivedStateProviderFactory(
   cache: { derivedStateProvider?: DerivedStateProvider } & CachedServices,
@@ -22,6 +22,6 @@ export async function derivedStateProviderFactory(
     "derivedStateProvider",
     opts,
     async () =>
-      new BackgroundDerivedStateProvider(await observableMemoryStorageServiceFactory(cache, opts)),
+      new BackgroundDerivedStateProvider(await storageServiceProviderFactory(cache, opts)),
   );
 }

apps/browser/src/platform/services/browser-memory-storage.service.ts

@@ -1,7 +1,16 @@
+import { AbstractMemoryStorageService } from "@bitwarden/common/platform/abstractions/storage.service";
+
 import AbstractChromeStorageService from "./abstractions/abstract-chrome-storage-api.service";
 
-export default class BrowserMemoryStorageService extends AbstractChromeStorageService {
+export default class BrowserMemoryStorageService
+  extends AbstractChromeStorageService
+  implements AbstractMemoryStorageService
+{
   constructor() {
     super(chrome.storage.session);
   }
+  type = "MemoryStorageService" as const;
+  getBypassCache<T>(key: string): Promise<T> {
+    return this.get(key);
+  }
 }

apps/browser/src/platform/state/background-derived-state.provider.ts

@@ -1,5 +1,9 @@
 import { Observable } from "rxjs";
 
+import {
+  AbstractStorageService,
+  ObservableStorageService,
+} from "@bitwarden/common/platform/abstractions/storage.service";
 import { DeriveDefinition, DerivedState } from "@bitwarden/common/platform/state";
 // eslint-disable-next-line import/no-restricted-paths -- extending this class for this client
 import { DefaultDerivedStateProvider } from "@bitwarden/common/platform/state/implementations/default-derived-state.provider";
@@ -12,11 +16,14 @@ export class BackgroundDerivedStateProvider extends DefaultDerivedStateProvider
     parentState$: Observable<TFrom>,
     deriveDefinition: DeriveDefinition<TFrom, TTo, TDeps>,
     dependencies: TDeps,
+    storageLocation: [string, AbstractStorageService & ObservableStorageService],
   ): DerivedState<TTo> {
+    const [cacheKey, storageService] = storageLocation;
     return new BackgroundDerivedState(
       parentState$,
       deriveDefinition,
-      this.memoryStorage,
+      storageService,
+      cacheKey,
       dependencies,
     );
   }

apps/browser/src/platform/state/background-derived-state.ts

@@ -23,10 +23,10 @@ export class BackgroundDerivedState<
     parentState$: Observable<TFrom>,
     deriveDefinition: DeriveDefinition<TFrom, TTo, TDeps>,
     memoryStorage: AbstractStorageService & ObservableStorageService,
+    portName: string,
     dependencies: TDeps,
   ) {
     super(parentState$, deriveDefinition, memoryStorage, dependencies);
-    const portName = deriveDefinition.buildCacheKey();
 
     // listen for foreground derived states to connect
     BrowserApi.addListener(chrome.runtime.onConnect, (port) => {

apps/browser/src/platform/state/derived-state-interactions.spec.ts

@@ -38,14 +38,21 @@ describe("foreground background derived state interactions", () => {
   let memoryStorage: FakeStorageService;
   const initialParent = "2020-01-01";
   const ngZone = mock<NgZone>();
+  const portName = "testPort";
 
   beforeEach(() => {
     mockPorts();
     parentState$ = new Subject<string>();
     memoryStorage = new FakeStorageService();
 
-    background = new BackgroundDerivedState(parentState$, deriveDefinition, memoryStorage, {});
-    foreground = new ForegroundDerivedState(deriveDefinition, memoryStorage, ngZone);
+    background = new BackgroundDerivedState(
+      parentState$,
+      deriveDefinition,
+      memoryStorage,
+      portName,
+      {},
+    );
+    foreground = new ForegroundDerivedState(deriveDefinition, memoryStorage, portName, ngZone);
   });
 
   afterEach(() => {
@@ -65,7 +72,12 @@ describe("foreground background derived state interactions", () => {
   });
 
   it("should initialize a late-connected foreground", async () => {
-    const newForeground = new ForegroundDerivedState(deriveDefinition, memoryStorage, ngZone);
+    const newForeground = new ForegroundDerivedState(
+      deriveDefinition,
+      memoryStorage,
+      portName,
+      ngZone,
+    );
     const backgroundEmissions = trackEmissions(background.state$);
     parentState$.next(initialParent);
     await awaitAsync();
@@ -82,8 +94,6 @@ describe("foreground background derived state interactions", () => {
       const dateString = "2020-12-12";
       const emissions = trackEmissions(background.state$);
 
-      // FIXME: Verify that this floating promise is intentional. If it is, add an explanatory comment and ensure there is proper error handling.
-      // eslint-disable-next-line @typescript-eslint/no-floating-promises
       await foreground.forceValue(new Date(dateString));
       await awaitAsync();
 
@@ -99,9 +109,7 @@ describe("foreground background derived state interactions", () => {
 
       expect(foreground["port"]).toBeDefined();
       const newDate = new Date();
-      // FIXME: Verify that this floating promise is intentional. If it is, add an explanatory comment and ensure there is proper error handling.
-      // eslint-disable-next-line @typescript-eslint/no-floating-promises
-      foreground.forceValue(newDate);
+      await foreground.forceValue(newDate);
       await awaitAsync();
 
       expect(connectMock.mock.calls.length).toBe(initialConnectCalls);
@@ -114,9 +122,7 @@ describe("foreground background derived state interactions", () => {
 
       expect(foreground["port"]).toBeUndefined();
       const newDate = new Date();
-      // FIXME: Verify that this floating promise is intentional. If it is, add an explanatory comment and ensure there is proper error handling.
-      // eslint-disable-next-line @typescript-eslint/no-floating-promises
-      foreground.forceValue(newDate);
+      await foreground.forceValue(newDate);
       await awaitAsync();
 
       expect(connectMock.mock.calls.length).toBe(initialConnectCalls + 1);

apps/browser/src/platform/state/foreground-derived-state.provider.ts

@@ -5,6 +5,7 @@ import {
   AbstractStorageService,
   ObservableStorageService,
 } from "@bitwarden/common/platform/abstractions/storage.service";
+import { StorageServiceProvider } from "@bitwarden/common/platform/services/storage-service.provider";
 import { DeriveDefinition, DerivedState } from "@bitwarden/common/platform/state";
 // eslint-disable-next-line import/no-restricted-paths -- extending this class for this client
 import { DefaultDerivedStateProvider } from "@bitwarden/common/platform/state/implementations/default-derived-state.provider";
@@ -14,16 +15,18 @@ import { ForegroundDerivedState } from "./foreground-derived-state";
 
 export class ForegroundDerivedStateProvider extends DefaultDerivedStateProvider {
   constructor(
-    memoryStorage: AbstractStorageService & ObservableStorageService,
+    storageServiceProvider: StorageServiceProvider,
     private ngZone: NgZone,
   ) {
-    super(memoryStorage);
+    super(storageServiceProvider);
   }
   override buildDerivedState<TFrom, TTo, TDeps extends DerivedStateDependencies>(
     _parentState$: Observable<TFrom>,
     deriveDefinition: DeriveDefinition<TFrom, TTo, TDeps>,
     _dependencies: TDeps,
+    storageLocation: [string, AbstractStorageService & ObservableStorageService],
   ): DerivedState<TTo> {
-    return new ForegroundDerivedState(deriveDefinition, this.memoryStorage, this.ngZone);
+    const [cacheKey, storageService] = storageLocation;
+    return new ForegroundDerivedState(deriveDefinition, storageService, cacheKey, this.ngZone);
   }
 }

apps/browser/src/platform/state/foreground-derived-state.spec.ts

@@ -33,13 +33,14 @@ jest.mock("../browser/run-inside-angular.operator", () => {
 describe("ForegroundDerivedState", () => {
   let sut: ForegroundDerivedState<Date>;
   let memoryStorage: FakeStorageService;
+  const portName = "testPort";
   const ngZone = mock<NgZone>();
 
   beforeEach(() => {
     memoryStorage = new FakeStorageService();
     memoryStorage.internalUpdateValuesRequireDeserialization(true);
     mockPorts();
-    sut = new ForegroundDerivedState(deriveDefinition, memoryStorage, ngZone);
+    sut = new ForegroundDerivedState(deriveDefinition, memoryStorage, portName, ngZone);
   });
 
   afterEach(() => {

apps/browser/src/platform/state/foreground-derived-state.ts

@@ -35,6 +35,7 @@ export class ForegroundDerivedState<TTo> implements DerivedState<TTo> {
   constructor(
     private deriveDefinition: DeriveDefinition<unknown, TTo, DerivedStateDependencies>,
     private memoryStorage: AbstractStorageService & ObservableStorageService,
+    private portName: string,
     private ngZone: NgZone,
   ) {
     this.storageKey = deriveDefinition.storageKey;
@@ -88,7 +89,7 @@ export class ForegroundDerivedState<TTo> implements DerivedState<TTo> {
       return;
     }
 
-    this.port = chrome.runtime.connect({ name: this.deriveDefinition.buildCacheKey() });
+    this.port = chrome.runtime.connect({ name: this.portName });
 
     this.backgroundResponses$ = fromChromeEvent(this.port.onMessage).pipe(
       map(([message]) => message as DerivedStateMessage),

apps/browser/src/platform/storage/browser-storage-service.provider.ts

@@ -0,0 +1,35 @@
+import {
+  AbstractStorageService,
+  ObservableStorageService,
+} from "@bitwarden/common/platform/abstractions/storage.service";
+import {
+  PossibleLocation,
+  StorageServiceProvider,
+} from "@bitwarden/common/platform/services/storage-service.provider";
+// eslint-disable-next-line import/no-restricted-paths
+import { ClientLocations } from "@bitwarden/common/platform/state/state-definition";
+
+export class BrowserStorageServiceProvider extends StorageServiceProvider {
+  constructor(
+    diskStorageService: AbstractStorageService & ObservableStorageService,
+    limitedMemoryStorageService: AbstractStorageService & ObservableStorageService,
+    private largeObjectMemoryStorageService: AbstractStorageService & ObservableStorageService,
+  ) {
+    super(diskStorageService, limitedMemoryStorageService);
+  }
+
+  override get(
+    defaultLocation: PossibleLocation,
+    overrides: Partial<ClientLocations>,
+  ): [location: PossibleLocation, service: AbstractStorageService & ObservableStorageService] {
+    const location = overrides["browser"] ?? defaultLocation;
+    switch (location) {
+      case "memory-large-object":
+        return ["memory-large-object", this.largeObjectMemoryStorageService];
+      default:
+        // Pass in computed location to super because they could have
+        // override default "disk" with web "memory".
+        return super.get(location, overrides);
+    }
+  }
+}

apps/browser/src/popup/services/services.module.ts

@@ -71,6 +71,7 @@ import { GlobalState } from "@bitwarden/common/platform/models/domain/global-sta
 import { ConsoleLogService } from "@bitwarden/common/platform/services/console-log.service";
 import { ContainerService } from "@bitwarden/common/platform/services/container.service";
 import { MigrationRunner } from "@bitwarden/common/platform/services/migration-runner";
+import { StorageServiceProvider } from "@bitwarden/common/platform/services/storage-service.provider";
 import { WebCryptoFunctionService } from "@bitwarden/common/platform/services/web-crypto-function.service";
 import {
   DerivedStateProvider,
@@ -108,6 +109,7 @@ import { DefaultBrowserStateService } from "../../platform/services/default-brow
 import I18nService from "../../platform/services/i18n.service";
 import { ForegroundPlatformUtilsService } from "../../platform/services/platform-utils/foreground-platform-utils.service";
 import { ForegroundDerivedStateProvider } from "../../platform/state/foreground-derived-state.provider";
+import { BrowserStorageServiceProvider } from "../../platform/storage/browser-storage-service.provider";
 import { ForegroundMemoryStorageService } from "../../platform/storage/foreground-memory-storage.service";
 import { fromChromeRuntimeMessaging } from "../../platform/utils/from-chrome-runtime-messaging";
 import { BrowserSendStateService } from "../../tools/popup/services/browser-send-state.service";
@@ -120,6 +122,10 @@ import { InitService } from "./init.service";
 import { PopupCloseWarningService } from "./popup-close-warning.service";
 import { PopupSearchService } from "./popup-search.service";
 
+const OBSERVABLE_LARGE_OBJECT_MEMORY_STORAGE = new SafeInjectionToken<
+  AbstractStorageService & ObservableStorageService
+>("OBSERVABLE_LARGE_OBJECT_MEMORY_STORAGE");
+
 const needsBackgroundInit = BrowserPopupUtils.backgroundInitializationRequired();
 const isPrivateMode = BrowserPopupUtils.inPrivateMode();
 const mainBackground: MainBackground = needsBackgroundInit
@@ -380,6 +386,21 @@ const safeProviders: SafeProvider[] = [
     },
     deps: [],
   }),
+  safeProvider({
+    provide: OBSERVABLE_LARGE_OBJECT_MEMORY_STORAGE,
+    useFactory: (
+      regularMemoryStorageService: AbstractMemoryStorageService & ObservableStorageService,
+    ) => {
+      if (BrowserApi.isManifestVersion(2)) {
+        return regularMemoryStorageService;
+      }
+
+      return getBgService<AbstractStorageService & ObservableStorageService>(
+        "largeObjectMemoryStorageForStateProviders",
+      )();
+    },
+    deps: [OBSERVABLE_MEMORY_STORAGE],
+  }),
   safeProvider({
     provide: OBSERVABLE_DISK_STORAGE,
     useExisting: AbstractStorageService,
@@ -466,7 +487,7 @@ const safeProviders: SafeProvider[] = [
   safeProvider({
     provide: DerivedStateProvider,
     useClass: ForegroundDerivedStateProvider,
-    deps: [OBSERVABLE_MEMORY_STORAGE, NgZone],
+    deps: [StorageServiceProvider, NgZone],
   }),
   safeProvider({
     provide: AutofillSettingsServiceAbstraction,
@@ -542,6 +563,15 @@ const safeProviders: SafeProvider[] = [
     },
     deps: [],
   }),
+  safeProvider({
+    provide: StorageServiceProvider,
+    useClass: BrowserStorageServiceProvider,
+    deps: [
+      OBSERVABLE_DISK_STORAGE,
+      OBSERVABLE_MEMORY_STORAGE,
+      OBSERVABLE_LARGE_OBJECT_MEMORY_STORAGE,
+    ],
+  }),
 ];
 
 @NgModule({

apps/cli/src/bw.ts

@@ -309,9 +309,7 @@ export class Main {
       this.singleUserStateProvider,
     );
 
-    this.derivedStateProvider = new DefaultDerivedStateProvider(
-      this.memoryStorageForStateProviders,
-    );
+    this.derivedStateProvider = new DefaultDerivedStateProvider(storageServiceProvider);
 
     this.stateProvider = new DefaultStateProvider(
       this.activeUserStateProvider,

apps/desktop/src/main.ts

@@ -157,7 +157,7 @@ export class Main {
       activeUserStateProvider,
       singleUserStateProvider,
       globalStateProvider,
-      new DefaultDerivedStateProvider(this.memoryStorageForStateProviders),
+      new DefaultDerivedStateProvider(storageServiceProvider),
     );
 
     this.environmentService = new DefaultEnvironmentService(stateProvider, accountService);