[PS-1854] Split services between background and visualizations (#4075)

* Elevate Map <-> Record JSON helpers to Utils

* Build Account from a StateService provided AccountDeserializer

* Allow Manifest V2 usage of session sync

Expands use of SessionSyncer to all Subject types. Correctly handles
replay buffer for each type to ignore the flood of data upon
subscription to each Subject type.

* Create browser-synced Policy Service

* Move BrowserFolderService

* Libs account serialization improvements

* Serialize Browser Accounts

* Separate StateService in background/visualizations

Visualizer state services share storages with background page, which
nicely emulates mv3 synchronization through session/local storage. There
should not be multithreading issues since all of these services are
still running through a single thread, we just now have multiple places
we are reading/writing data from.

Smaller improvements
* Rename browser's state service to BrowserStateService
* Remove unused WithPrototype decorator :celebrate:
* Removed conversion on withPrototypeForArrayMembers. It's reasonable to
think that if the type is maintained, it doesn't need conversion.

Eventually, we should be able to remove the withPrototypeForArrayMembers
decorator as well, but that will require a bit more work on
(de)serialization of the Accounts.data property.

* Make Record <-> Map idempotent

Should we get in a situation where we _think_ an object has been
jsonified, but hasn't been, we need to correctly deal with the object
received to create our target.

* Check all requirements while duck typing

* Name client services after the client

* Use union type to limit initialize options

* Fixup usages of `initializeAs`

* Add OrganizationService to synced services

Co-Authored-By: Daniel James Smith <djsmith85@users.noreply.github.com>

* Add Settings service to synced services

Co-Authored-By: Daniel James Smith <djsmith85@users.noreply.github.com>

* Add missing BrowserStateService

* Fix factories to use browser-specific service overides

* Fix org-service registration in services.module

* Revert "Add missing BrowserStateService"

This reverts commit 81cf384e87.

* Fix session syncer tests

* Fix synced item metadata tests

* Early return null json objects

* Prefer abstract service dependencies

* Prefer minimal browser service overrides

* [SG-632] - Change forwarded providers radio buttons list to dropdown (#4045)

* SG-632 - Changed forwarded providers list of radio buttons to dropdown

* SG-632 - Added role attributes to improve accessibility.

* SG-632 - Added sorting to array and empty option

* SG-632 - Fix styling to match standards.

* rename cipehrs component to vault items component (#4081)

* Update the version hash for the QA Web build artifact to follow SemVer syntax (#4102)

* Remove extra call to toJSON() (#4101)

Co-authored-by: Daniel James Smith <djsmith85@users.noreply.github.com>
Co-authored-by: Daniel James Smith <djsmith@web.de>
Co-authored-by: Carlos Gonçalves <carlosmaccam@gmail.com>
Co-authored-by: Jake Fink <jfink@bitwarden.com>
Co-authored-by: Joseph Flinn <58369717+joseph-flinn@users.noreply.github.com>
Co-authored-by: Robyn MacCallum <robyntmaccallum@gmail.com>

libs/common/src/misc/utils.ts

@@ -1,5 +1,6 @@
 /* eslint-disable no-useless-escape */
 import { getHostname, parse } from "tldts";
+import { Merge } from "type-fest";
 
 import { CryptoService } from "../abstractions/crypto.service";
 import { EncryptService } from "../abstractions/encrypt.service";
@@ -55,6 +56,10 @@ export class Utils {
   }
 
   static fromB64ToArray(str: string): Uint8Array {
+    if (str == null) {
+      return null;
+    }
+
     if (Utils.isNode) {
       return new Uint8Array(Buffer.from(str, "base64"));
     } else {
@@ -108,6 +113,9 @@ export class Utils {
   }
 
   static fromBufferToB64(buffer: ArrayBuffer): string {
+    if (buffer == null) {
+      return null;
+    }
     if (Utils.isNode) {
       return Buffer.from(buffer).toString("base64");
     } else {
@@ -423,6 +431,57 @@ export class Utils {
     return this.global.bitwardenContainerService;
   }
 
+  /**
+   * Converts map to a Record<string, V> with the same data. Inverse of recordToMap
+   * Useful in toJSON methods, since Maps are not serializable
+   * @param map
+   * @returns
+   */
+  static mapToRecord<K extends string | number, V>(map: Map<K, V>): Record<string, V> {
+    if (map == null) {
+      return null;
+    }
+    if (!(map instanceof Map)) {
+      return map;
+    }
+    return Object.fromEntries(map);
+  }
+
+  /**
+   * Converts record to a Map<string, V> with the same data. Inverse of mapToRecord
+   * Useful in fromJSON methods, since Maps are not serializable
+   *
+   * Warning: If the record has string keys that are numbers, they will be converted to numbers in the map
+   * @param record
+   * @returns
+   */
+  static recordToMap<K extends string | number, V>(record: Record<K, V>): Map<K, V> {
+    if (record == null) {
+      return null;
+    } else if (record instanceof Map) {
+      return record;
+    }
+
+    const entries = Object.entries(record);
+    if (entries.length === 0) {
+      return new Map();
+    }
+
+    if (isNaN(Number(entries[0][0]))) {
+      return new Map(entries) as Map<K, V>;
+    } else {
+      return new Map(entries.map((e) => [Number(e[0]), e[1]])) as Map<K, V>;
+    }
+  }
+
+  /** Applies Object.assign, but converts the type nicely using Type-Fest Merge<Destination, Source> */
+  static merge<Destination, Source>(
+    destination: Destination,
+    source: Source
+  ): Merge<Destination, Source> {
+    return Object.assign(destination, source) as unknown as Merge<Destination, Source>;
+  }
+
   private static isMobile(win: Window) {
     let mobile = false;
     ((a) => {

libs/common/src/models/domain/account.ts

@@ -1,4 +1,4 @@
-import { Except, Jsonify } from "type-fest";
+import { Jsonify } from "type-fest";
 
 import { AuthenticationStatus } from "../../enums/authenticationStatus";
 import { KdfType } from "../../enums/kdfType";
@@ -40,7 +40,7 @@ export class EncryptionPair<TEncrypted, TDecrypted> {
   }
 
   static fromJSON<TEncrypted, TDecrypted>(
-    obj: Jsonify<EncryptionPair<Jsonify<TEncrypted>, Jsonify<TDecrypted>>>,
+    obj: { encrypted?: Jsonify<TEncrypted>; decrypted?: string | Jsonify<TDecrypted> },
     decryptedFromJson?: (decObj: Jsonify<TDecrypted> | string) => TDecrypted,
     encryptedFromJson?: (encObj: Jsonify<TEncrypted>) => TEncrypted
   ) {
@@ -123,7 +123,7 @@ export class AccountKeys {
   apiKeyClientSecret?: string;
 
   toJSON() {
-    return Object.assign(this as Except<AccountKeys, "publicKey">, {
+    return Utils.merge(this, {
       publicKey: Utils.fromBufferToByteString(this.publicKey),
     });
   }
@@ -251,7 +251,7 @@ export class AccountSettings {
 }
 
 export type AccountSettingsSettings = {
-  equivalentDomains?: { [id: string]: any };
+  equivalentDomains?: string[][];
 };
 
 export class AccountTokens {

libs/common/src/models/domain/organization.ts

@@ -1,3 +1,5 @@
+import { Jsonify } from "type-fest";
+
 import { OrganizationUserStatusType } from "../../enums/organizationUserStatusType";
 import { OrganizationUserType } from "../../enums/organizationUserType";
 import { ProductType } from "../../enums/productType";
@@ -201,4 +203,15 @@ export class Organization {
   get hasProvider() {
     return this.providerId != null || this.providerName != null;
   }
+
+  static fromJSON(json: Jsonify<Organization>) {
+    if (json == null) {
+      return null;
+    }
+
+    return Object.assign(new Organization(), json, {
+      familySponsorshipLastSyncDate: new Date(json.familySponsorshipLastSyncDate),
+      familySponsorshipValidUntil: new Date(json.familySponsorshipValidUntil),
+    });
+  }
 }

libs/common/src/models/domain/state.spec.ts

@@ -4,22 +4,25 @@ import { State } from "./state";
 describe("state", () => {
   describe("fromJSON", () => {
     it("should deserialize to an instance of itself", () => {
-      expect(State.fromJSON({})).toBeInstanceOf(State);
+      expect(State.fromJSON({}, () => new Account({}))).toBeInstanceOf(State);
     });
 
     it("should always assign an object to accounts", () => {
-      const state = State.fromJSON({});
+      const state = State.fromJSON({}, () => new Account({}));
       expect(state.accounts).not.toBeNull();
       expect(state.accounts).toEqual({});
     });
 
     it("should build an account map", () => {
       const accountsSpy = jest.spyOn(Account, "fromJSON");
-      const state = State.fromJSON({
-        accounts: {
-          userId: {},
+      const state = State.fromJSON(
+        {
+          accounts: {
+            userId: {},
+          },
         },
-      });
+        Account.fromJSON
+      );
 
       expect(state.accounts["userId"]).toBeInstanceOf(Account);
       expect(accountsSpy).toHaveBeenCalled();

libs/common/src/models/domain/state.ts

@@ -19,26 +19,28 @@ export class State<
 
   // TODO, make Jsonify<State,TGlobalState,TAccount> work. It currently doesn't because Globals doesn't implement Jsonify.
   static fromJSON<TGlobalState extends GlobalState, TAccount extends Account>(
-    obj: any
+    obj: any,
+    accountDeserializer: (json: Jsonify<TAccount>) => TAccount
   ): State<TGlobalState, TAccount> {
     if (obj == null) {
       return null;
     }
 
     return Object.assign(new State(null), obj, {
-      accounts: State.buildAccountMapFromJSON(obj?.accounts),
+      accounts: State.buildAccountMapFromJSON(obj?.accounts, accountDeserializer),
     });
   }
 
-  private static buildAccountMapFromJSON(
-    jsonAccounts: Jsonify<{ [userId: string]: Jsonify<Account> }>
+  private static buildAccountMapFromJSON<TAccount extends Account>(
+    jsonAccounts: { [userId: string]: Jsonify<TAccount> },
+    accountDeserializer: (json: Jsonify<TAccount>) => TAccount
   ) {
     if (!jsonAccounts) {
       return {};
     }
-    const accounts: { [userId: string]: Account } = {};
+    const accounts: { [userId: string]: TAccount } = {};
     for (const userId in jsonAccounts) {
-      accounts[userId] = Account.fromJSON(jsonAccounts[userId]);
+      accounts[userId] = accountDeserializer(jsonAccounts[userId]);
     }
     return accounts;
   }

libs/common/src/models/view/send-file.view.ts

@@ -1,3 +1,4 @@
+import { DeepJsonify } from "../../types/deep-jsonify";
 import { SendFile } from "../domain/send-file";
 
 import { View } from "./view";
@@ -28,4 +29,12 @@ export class SendFileView implements View {
     }
     return 0;
   }
+
+  static fromJSON(json: DeepJsonify<SendFileView>) {
+    if (json == null) {
+      return null;
+    }
+
+    return Object.assign(new SendFileView(), json);
+  }
 }

libs/common/src/models/view/send-text.view.ts

@@ -1,3 +1,4 @@
+import { DeepJsonify } from "../../types/deep-jsonify";
 import { SendText } from "../domain/send-text";
 
 import { View } from "./view";
@@ -17,4 +18,12 @@ export class SendTextView implements View {
   get maskedText(): string {
     return this.text != null ? "••••••••" : null;
   }
+
+  static fromJSON(json: DeepJsonify<SendTextView>) {
+    if (json == null) {
+      return null;
+    }
+
+    return Object.assign(new SendTextView(), json);
+  }
 }

libs/common/src/models/view/send.view.ts

@@ -1,5 +1,6 @@
 import { SendType } from "../../enums/sendType";
 import { Utils } from "../../misc/utils";
+import { DeepJsonify } from "../../types/deep-jsonify";
 import { Send } from "../domain/send";
 import { SymmetricCryptoKey } from "../domain/symmetric-crypto-key";
 
@@ -65,4 +66,26 @@ export class SendView implements View {
   get pendingDelete(): boolean {
     return this.deletionDate <= new Date();
   }
+
+  toJSON() {
+    return Utils.merge(this, {
+      key: Utils.fromBufferToB64(this.key),
+    });
+  }
+
+  static fromJSON(json: DeepJsonify<SendView>) {
+    if (json == null) {
+      return null;
+    }
+
+    return Object.assign(new SendView(), json, {
+      key: Utils.fromB64ToArray(json.key)?.buffer,
+      cryptoKey: SymmetricCryptoKey.fromJSON(json.cryptoKey),
+      text: SendTextView.fromJSON(json.text),
+      file: SendFileView.fromJSON(json.file),
+      revisionDate: json.revisionDate == null ? null : new Date(json.revisionDate),
+      deletionDate: json.deletionDate == null ? null : new Date(json.deletionDate),
+      expirationDate: json.expirationDate == null ? null : new Date(json.expirationDate),
+    });
+  }
 }

libs/common/src/services/cipher.service.ts

@@ -412,7 +412,7 @@ export class CipherService implements CipherServiceAbstraction {
         : firstValueFrom(this.settingsService.settings$).then(
             (settings: AccountSettingsSettings) => {
               let matches: any[] = [];
-              settings.equivalentDomains?.forEach((eqDomain: any) => {
+              settings?.equivalentDomains?.forEach((eqDomain: any) => {
                 if (eqDomain.length && eqDomain.indexOf(domain) >= 0) {
                   matches = matches.concat(eqDomain);
                 }

libs/common/src/services/organization/organization.service.ts

@@ -6,7 +6,7 @@ import { OrganizationData } from "../../models/data/organization.data";
 import { Organization } from "../../models/domain/organization";
 
 export class OrganizationService implements InternalOrganizationServiceAbstraction {
-  private _organizations = new BehaviorSubject<Organization[]>([]);
+  protected _organizations = new BehaviorSubject<Organization[]>([]);
 
   organizations$ = this._organizations.asObservable();
 

libs/common/src/services/policy/policy.service.ts

@@ -16,7 +16,7 @@ import { ListResponse } from "../../models/response/list.response";
 import { PolicyResponse } from "../../models/response/policy.response";
 
 export class PolicyService implements InternalPolicyServiceAbstraction {
-  private _policies: BehaviorSubject<Policy[]> = new BehaviorSubject([]);
+  protected _policies: BehaviorSubject<Policy[]> = new BehaviorSubject([]);
 
   policies$ = this._policies.asObservable();
 

libs/common/src/services/settings.service.ts

@@ -6,7 +6,7 @@ import { Utils } from "../misc/utils";
 import { AccountSettingsSettings } from "../models/domain/account";
 
 export class SettingsService implements SettingsServiceAbstraction {
-  private _settings: BehaviorSubject<AccountSettingsSettings> = new BehaviorSubject({});
+  protected _settings: BehaviorSubject<AccountSettingsSettings> = new BehaviorSubject({});
 
   settings$ = this._settings.asObservable();
 

libs/common/src/services/state.service.ts

@@ -1,4 +1,5 @@
 import { BehaviorSubject, concatMap } from "rxjs";
+import { Jsonify } from "type-fest";
 
 import { LogService } from "../abstractions/log.service";
 import { StateService as StateServiceAbstraction } from "../abstractions/state.service";
@@ -13,6 +14,7 @@ import { StorageLocation } from "../enums/storageLocation";
 import { ThemeType } from "../enums/themeType";
 import { UriMatchType } from "../enums/uriMatchType";
 import { StateFactory } from "../factories/stateFactory";
+import { Utils } from "../misc/utils";
 import { CipherData } from "../models/data/cipher.data";
 import { CollectionData } from "../models/data/collection.data";
 import { EncryptedOrganizationKeyData } from "../models/data/encrypted-organization-key.data";
@@ -65,13 +67,13 @@ export class StateService<
   TAccount extends Account = Account
 > implements StateServiceAbstraction<TAccount>
 {
-  private accountsSubject = new BehaviorSubject<{ [userId: string]: TAccount }>({});
+  protected accountsSubject = new BehaviorSubject<{ [userId: string]: TAccount }>({});
   accounts$ = this.accountsSubject.asObservable();
 
-  private activeAccountSubject = new BehaviorSubject<string | null>(null);
+  protected activeAccountSubject = new BehaviorSubject<string | null>(null);
   activeAccount$ = this.activeAccountSubject.asObservable();
 
-  private activeAccountUnlockedSubject = new BehaviorSubject<boolean>(false);
+  protected activeAccountUnlockedSubject = new BehaviorSubject<boolean>(false);
   activeAccountUnlocked$ = this.activeAccountUnlockedSubject.asObservable();
 
   private hasBeenInited = false;
@@ -79,6 +81,9 @@ export class StateService<
 
   private accountDiskCache = new Map<string, TAccount>();
 
+  // default account serializer, must be overridden by child class
+  protected accountDeserializer = Account.fromJSON as (json: Jsonify<TAccount>) => TAccount;
+
   constructor(
     protected storageService: AbstractStorageService,
     protected secureStorageService: AbstractStorageService,
@@ -676,7 +681,7 @@ export class StateService<
     const account = await this.getAccount(
       this.reconcileOptions(options, await this.defaultInMemoryOptions())
     );
-    return this.recordToMap(account?.keys?.organizationKeys?.decrypted);
+    return Utils.recordToMap(account?.keys?.organizationKeys?.decrypted);
   }
 
   async setDecryptedOrganizationKeys(
@@ -686,7 +691,7 @@ export class StateService<
     const account = await this.getAccount(
       this.reconcileOptions(options, await this.defaultInMemoryOptions())
     );
-    account.keys.organizationKeys.decrypted = this.mapToRecord(value);
+    account.keys.organizationKeys.decrypted = Utils.mapToRecord(value);
     await this.saveAccount(
       account,
       this.reconcileOptions(options, await this.defaultInMemoryOptions())
@@ -774,7 +779,7 @@ export class StateService<
     const account = await this.getAccount(
       this.reconcileOptions(options, await this.defaultInMemoryOptions())
     );
-    return this.recordToMap(account?.keys?.providerKeys?.decrypted);
+    return Utils.recordToMap(account?.keys?.providerKeys?.decrypted);
   }
 
   async setDecryptedProviderKeys(
@@ -784,7 +789,7 @@ export class StateService<
     const account = await this.getAccount(
       this.reconcileOptions(options, await this.defaultInMemoryOptions())
     );
-    account.keys.providerKeys.decrypted = this.mapToRecord(value);
+    account.keys.providerKeys.decrypted = Utils.mapToRecord(value);
     await this.saveAccount(
       account,
       this.reconcileOptions(options, await this.defaultInMemoryOptions())
@@ -2744,7 +2749,7 @@ export class StateService<
 
   protected async state(): Promise<State<TGlobalState, TAccount>> {
     const state = await this.memoryStorageService.get<State<TGlobalState, TAccount>>(keys.state, {
-      deserializer: (s) => State.fromJSON(s),
+      deserializer: (s) => State.fromJSON(s, this.accountDeserializer),
     });
     return state;
   }
@@ -2765,50 +2770,6 @@ export class StateService<
       await this.setState(updatedState);
     });
   }
-
-  private mapToRecord<V>(map: Map<string, V>): Record<string, V> {
-    return map == null ? null : Object.fromEntries(map);
-  }
-
-  private recordToMap<V>(record: Record<string, V>): Map<string, V> {
-    return record == null ? null : new Map(Object.entries(record));
-  }
-}
-
-export function withPrototype<T>(
-  constructor: new (...args: any[]) => T,
-  converter: (input: any) => T = (i) => i
-): (
-  target: any,
-  propertyKey: string | symbol,
-  descriptor: PropertyDescriptor
-) => { value: (...args: any[]) => Promise<T> } {
-  return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
-    const originalMethod = descriptor.value;
-
-    return {
-      value: function (...args: any[]) {
-        const originalResult: Promise<T> = originalMethod.apply(this, args);
-
-        if (!(originalResult instanceof Promise)) {
-          throw new Error(
-            `Error applying prototype to stored value -- result is not a promise for method ${String(
-              propertyKey
-            )}`
-          );
-        }
-
-        return originalResult.then((result) => {
-          return result == null ||
-            result.constructor.name === constructor.prototype.constructor.name
-            ? converter(result as T)
-            : converter(
-                Object.create(constructor.prototype, Object.getOwnPropertyDescriptors(result)) as T
-              );
-        });
-      },
-    };
-  };
 }
 
 function withPrototypeForArrayMembers<T>(
@@ -2847,7 +2808,7 @@ function withPrototypeForArrayMembers<T>(
             return result.map((r) => {
               return r == null ||
                 r.constructor.name === memberConstructor.prototype.constructor.name
-                ? memberConverter(r)
+                ? r
                 : memberConverter(
                     Object.create(memberConstructor.prototype, Object.getOwnPropertyDescriptors(r))
                   );