[PM-30718] add IndexedDB storage service for phishing data (#18344)

* add PhishingIndexedDbService for IndexedDB storage Add a dedicated IndexedDB storage service for phishing detection data. This service provides save, load, and clear operations using IndexedDB instead of chrome.storage.local to avoid broadcast overhead, size limitations, and JSON serialization cost for large datasets. * add unit tests for PhishingIndexedDbService Add comprehensive tests for save, load, and clear operations with mocked IndexedDB. Tests cover success cases, error handling, and database initialization with object store creation. * add PhishingIndexedDbService core structure - Add IndexedDB service with per-operation database opening - Define PhishingUrlRecord type for row storage - Include clearStore helper for atomic data replacement - Service worker safe: no cached connections * add saveUrls with chunked writes - Add PhishingUrlRecord type for row storage - Store each URL as individual row - Chunk writes at 50K per transaction for responsiveness - Atomic replacement: clear then save * add hasUrl for lookups - Direct IndexedDB index lookup via keyPath - Returns boolean, handles errors gracefully * add loadAllUrls with cursor iteration - Cursor-based bulk load for fallback scenarios - Memory-efficient: no intermediate array duplication - Returns empty array on error * add saveUrlsFromStream for memory efficiency - Stream directly from fetch response body - Parse newline-delimited URLs incrementally - Reuse chunked save infrastructure * update PhishingIndexedDbService tests - Replace blob-based tests with row-per-URL API tests - Test saveUrls, hasUrl, loadAllUrls, saveUrlsFromStream - Verify chunked writes and cursor iteration - Use stream/web ReadableStream with type cast for Node.js compatibility * use proper URL syntax and cleanup global state Update test data to use proper URL syntax with https:// prefix to match real phishing.database format. Add cleanup of global.indexedDB in afterEach to prevent test pollution. * improve stream processing correctness and efficiency - Move decoder.decode() before done check with { stream: !done } to flush properly - Use array reassignment instead of splice() for O(1) chunk clearing - Use single trim via local variable to avoid double-trim - Centralize URL cleaning in saveChunked(), simplify saveChunk() - Use explicit urls.length > 0 comparison * duplicate urls test * split final buffer by newlines
2026-01-29 15:53:45 +00:00 · 2026-01-21 15:55:14 -05:00
parent d5273c7abe
commit 1db601b82f
2 changed files with 616 additions and 0 deletions
--- a/apps/browser/src/dirt/phishing-detection/services/phishing-indexeddb.service.spec.ts
+++ b/apps/browser/src/dirt/phishing-detection/services/phishing-indexeddb.service.spec.ts
@@ -0,0 +1,375 @@
+import { ReadableStream as NodeReadableStream } from "stream/web";
+
+import { mock, MockProxy } from "jest-mock-extended";
+
+import { LogService } from "@bitwarden/common/platform/abstractions/log.service";
+
+import { PhishingIndexedDbService } from "./phishing-indexeddb.service";
+
+describe("PhishingIndexedDbService", () => {
+  let service: PhishingIndexedDbService;
+  let logService: MockProxy<LogService>;
+
+  // Mock IndexedDB storage (keyed by URL for row-per-URL storage)
+  let mockStore: Map<string, { url: string }>;
+  let mockObjectStore: any;
+  let mockTransaction: any;
+  let mockDb: any;
+  let mockOpenRequest: any;
+
+  beforeEach(() => {
+    logService = mock<LogService>();
+    mockStore = new Map();
+
+    // Mock IDBObjectStore
+    mockObjectStore = {
+      put: jest.fn().mockImplementation((record: { url: string }) => {
+        const request = {
+          error: null as DOMException | null,
+          result: undefined as undefined,
+          onsuccess: null as (() => void) | null,
+          onerror: null as (() => void) | null,
+        };
+        setTimeout(() => {
+          mockStore.set(record.url, record);
+          request.onsuccess?.();
+        }, 0);
+        return request;
+      }),
+      get: jest.fn().mockImplementation((key: string) => {
+        const request = {
+          error: null as DOMException | null,
+          result: mockStore.get(key),
+          onsuccess: null as (() => void) | null,
+          onerror: null as (() => void) | null,
+        };
+        setTimeout(() => {
+          request.result = mockStore.get(key);
+          request.onsuccess?.();
+        }, 0);
+        return request;
+      }),
+      clear: jest.fn().mockImplementation(() => {
+        const request = {
+          error: null as DOMException | null,
+          result: undefined as undefined,
+          onsuccess: null as (() => void) | null,
+          onerror: null as (() => void) | null,
+        };
+        setTimeout(() => {
+          mockStore.clear();
+          request.onsuccess?.();
+        }, 0);
+        return request;
+      }),
+      openCursor: jest.fn().mockImplementation(() => {
+        const entries = Array.from(mockStore.entries());
+        let index = 0;
+        const request = {
+          error: null as DOMException | null,
+          result: null as any,
+          onsuccess: null as ((e: any) => void) | null,
+          onerror: null as (() => void) | null,
+        };
+        const advanceCursor = () => {
+          if (index < entries.length) {
+            const [, value] = entries[index];
+            index++;
+            request.result = {
+              value,
+              continue: () => setTimeout(advanceCursor, 0),
+            };
+          } else {
+            request.result = null;
+          }
+          request.onsuccess?.({ target: request });
+        };
+        setTimeout(advanceCursor, 0);
+        return request;
+      }),
+    };
+
+    // Mock IDBTransaction
+    mockTransaction = {
+      objectStore: jest.fn().mockReturnValue(mockObjectStore),
+      oncomplete: null as (() => void) | null,
+      onerror: null as (() => void) | null,
+    };
+
+    // Trigger oncomplete after a tick
+    const originalObjectStore = mockTransaction.objectStore;
+    mockTransaction.objectStore = jest.fn().mockImplementation((...args: any[]) => {
+      setTimeout(() => mockTransaction.oncomplete?.(), 0);
+      return originalObjectStore(...args);
+    });
+
+    // Mock IDBDatabase
+    mockDb = {
+      transaction: jest.fn().mockReturnValue(mockTransaction),
+      close: jest.fn(),
+      objectStoreNames: {
+        contains: jest.fn().mockReturnValue(true),
+      },
+      createObjectStore: jest.fn(),
+    };
+
+    // Mock IDBOpenDBRequest
+    mockOpenRequest = {
+      error: null as DOMException | null,
+      result: mockDb,
+      onsuccess: null as (() => void) | null,
+      onerror: null as (() => void) | null,
+      onupgradeneeded: null as ((event: any) => void) | null,
+    };
+
+    // Mock indexedDB.open
+    const mockIndexedDB = {
+      open: jest.fn().mockImplementation(() => {
+        setTimeout(() => {
+          mockOpenRequest.onsuccess?.();
+        }, 0);
+        return mockOpenRequest;
+      }),
+    };
+
+    global.indexedDB = mockIndexedDB as any;
+
+    service = new PhishingIndexedDbService(logService);
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+    delete (global as any).indexedDB;
+  });
+
+  describe("saveUrls", () => {
+    it("stores URLs in IndexedDB and returns true", async () => {
+      const urls = ["https://phishing.com", "https://malware.net"];
+
+      const result = await service.saveUrls(urls);
+
+      expect(result).toBe(true);
+      expect(mockDb.transaction).toHaveBeenCalledWith("phishing-urls", "readwrite");
+      expect(mockObjectStore.clear).toHaveBeenCalled();
+      expect(mockObjectStore.put).toHaveBeenCalledTimes(2);
+      expect(mockDb.close).toHaveBeenCalled();
+    });
+
+    it("handles empty array", async () => {
+      const result = await service.saveUrls([]);
+
+      expect(result).toBe(true);
+      expect(mockObjectStore.clear).toHaveBeenCalled();
+    });
+
+    it("trims whitespace from URLs", async () => {
+      const urls = ["  https://example.com  ", "\nhttps://test.org\n"];
+
+      await service.saveUrls(urls);
+
+      expect(mockObjectStore.put).toHaveBeenCalledWith({ url: "https://example.com" });
+      expect(mockObjectStore.put).toHaveBeenCalledWith({ url: "https://test.org" });
+    });
+
+    it("skips empty lines", async () => {
+      const urls = ["https://example.com", "", "  ", "https://test.org"];
+
+      await service.saveUrls(urls);
+
+      expect(mockObjectStore.put).toHaveBeenCalledTimes(2);
+    });
+
+    it("handles duplicate URLs via upsert (keyPath deduplication)", async () => {
+      const urls = [
+        "https://example.com",
+        "https://example.com", // duplicate
+        "https://test.org",
+      ];
+
+      const result = await service.saveUrls(urls);
+
+      expect(result).toBe(true);
+      // put() is called 3 times, but mockStore (using Map with URL as key)
+      // only stores 2 unique entries - demonstrating upsert behavior
+      expect(mockObjectStore.put).toHaveBeenCalledTimes(3);
+      expect(mockStore.size).toBe(2);
+    });
+
+    it("logs error and returns false on failure", async () => {
+      const error = new Error("IndexedDB error");
+      mockOpenRequest.error = error;
+      (global.indexedDB.open as jest.Mock).mockImplementation(() => {
+        setTimeout(() => {
+          mockOpenRequest.onerror?.();
+        }, 0);
+        return mockOpenRequest;
+      });
+
+      const result = await service.saveUrls(["https://test.com"]);
+
+      expect(result).toBe(false);
+      expect(logService.error).toHaveBeenCalledWith(
+        "[PhishingIndexedDbService] Save failed",
+        expect.any(Error),
+      );
+    });
+  });
+
+  describe("hasUrl", () => {
+    it("returns true for existing URL", async () => {
+      mockStore.set("https://example.com", { url: "https://example.com" });
+
+      const result = await service.hasUrl("https://example.com");
+
+      expect(result).toBe(true);
+      expect(mockDb.transaction).toHaveBeenCalledWith("phishing-urls", "readonly");
+      expect(mockObjectStore.get).toHaveBeenCalledWith("https://example.com");
+    });
+
+    it("returns false for non-existing URL", async () => {
+      const result = await service.hasUrl("https://notfound.com");
+
+      expect(result).toBe(false);
+    });
+
+    it("returns false on error", async () => {
+      const error = new Error("IndexedDB error");
+      mockOpenRequest.error = error;
+      (global.indexedDB.open as jest.Mock).mockImplementation(() => {
+        setTimeout(() => {
+          mockOpenRequest.onerror?.();
+        }, 0);
+        return mockOpenRequest;
+      });
+
+      const result = await service.hasUrl("https://example.com");
+
+      expect(result).toBe(false);
+      expect(logService.error).toHaveBeenCalledWith(
+        "[PhishingIndexedDbService] Check failed",
+        expect.any(Error),
+      );
+    });
+  });
+
+  describe("loadAllUrls", () => {
+    it("loads all URLs using cursor", async () => {
+      mockStore.set("https://example.com", { url: "https://example.com" });
+      mockStore.set("https://test.org", { url: "https://test.org" });
+
+      const result = await service.loadAllUrls();
+
+      expect(result).toContain("https://example.com");
+      expect(result).toContain("https://test.org");
+      expect(result.length).toBe(2);
+    });
+
+    it("returns empty array when no data exists", async () => {
+      const result = await service.loadAllUrls();
+
+      expect(result).toEqual([]);
+    });
+
+    it("returns empty array on error", async () => {
+      const error = new Error("IndexedDB error");
+      mockOpenRequest.error = error;
+      (global.indexedDB.open as jest.Mock).mockImplementation(() => {
+        setTimeout(() => {
+          mockOpenRequest.onerror?.();
+        }, 0);
+        return mockOpenRequest;
+      });
+
+      const result = await service.loadAllUrls();
+
+      expect(result).toEqual([]);
+      expect(logService.error).toHaveBeenCalledWith(
+        "[PhishingIndexedDbService] Load failed",
+        expect.any(Error),
+      );
+    });
+  });
+
+  describe("saveUrlsFromStream", () => {
+    it("saves URLs from stream", async () => {
+      const content = "https://example.com\nhttps://test.org\nhttps://phishing.net";
+      const stream = new NodeReadableStream({
+        start(controller) {
+          controller.enqueue(new TextEncoder().encode(content));
+          controller.close();
+        },
+      }) as unknown as ReadableStream<Uint8Array>;
+
+      const result = await service.saveUrlsFromStream(stream);
+
+      expect(result).toBe(true);
+      expect(mockObjectStore.clear).toHaveBeenCalled();
+      expect(mockObjectStore.put).toHaveBeenCalledTimes(3);
+    });
+
+    it("handles chunked stream data", async () => {
+      const content = "https://url1.com\nhttps://url2.com";
+      const encoder = new TextEncoder();
+      const encoded = encoder.encode(content);
+
+      // Split into multiple small chunks
+      const stream = new NodeReadableStream({
+        start(controller) {
+          controller.enqueue(encoded.slice(0, 5));
+          controller.enqueue(encoded.slice(5, 10));
+          controller.enqueue(encoded.slice(10));
+          controller.close();
+        },
+      }) as unknown as ReadableStream<Uint8Array>;
+
+      const result = await service.saveUrlsFromStream(stream);
+
+      expect(result).toBe(true);
+      expect(mockObjectStore.put).toHaveBeenCalledTimes(2);
+    });
+
+    it("returns false on error", async () => {
+      const error = new Error("IndexedDB error");
+      mockOpenRequest.error = error;
+      (global.indexedDB.open as jest.Mock).mockImplementation(() => {
+        setTimeout(() => {
+          mockOpenRequest.onerror?.();
+        }, 0);
+        return mockOpenRequest;
+      });
+
+      const stream = new NodeReadableStream({
+        start(controller) {
+          controller.enqueue(new TextEncoder().encode("https://test.com"));
+          controller.close();
+        },
+      }) as unknown as ReadableStream<Uint8Array>;
+
+      const result = await service.saveUrlsFromStream(stream);
+
+      expect(result).toBe(false);
+      expect(logService.error).toHaveBeenCalledWith(
+        "[PhishingIndexedDbService] Stream save failed",
+        expect.any(Error),
+      );
+    });
+  });
+
+  describe("database initialization", () => {
+    it("creates object store with keyPath on upgrade", async () => {
+      mockDb.objectStoreNames.contains.mockReturnValue(false);
+
+      (global.indexedDB.open as jest.Mock).mockImplementation(() => {
+        setTimeout(() => {
+          mockOpenRequest.onupgradeneeded?.({ target: mockOpenRequest });
+          mockOpenRequest.onsuccess?.();
+        }, 0);
+        return mockOpenRequest;
+      });
+
+      await service.hasUrl("https://test.com");
+
+      expect(mockDb.createObjectStore).toHaveBeenCalledWith("phishing-urls", { keyPath: "url" });
+    });
+  });
+});
--- a/apps/browser/src/dirt/phishing-detection/services/phishing-indexeddb.service.ts
+++ b/apps/browser/src/dirt/phishing-detection/services/phishing-indexeddb.service.ts
@@ -0,0 +1,241 @@
+import { LogService } from "@bitwarden/common/platform/abstractions/log.service";
+
+/**
+ * Record type for phishing URL storage in IndexedDB.
+ */
+type PhishingUrlRecord = { url: string };
+
+/**
+ * IndexedDB storage service for phishing URLs.
+ * Stores URLs as individual rows.
+ */
+export class PhishingIndexedDbService {
+  private readonly DB_NAME = "bitwarden-phishing";
+  private readonly STORE_NAME = "phishing-urls";
+  private readonly DB_VERSION = 1;
+  private readonly CHUNK_SIZE = 50000;
+
+  constructor(private logService: LogService) {}
+
+  /**
+   * Opens the IndexedDB database, creating the object store if needed.
+   */
+  private openDatabase(): Promise<IDBDatabase> {
+    return new Promise((resolve, reject) => {
+      const req = indexedDB.open(this.DB_NAME, this.DB_VERSION);
+      req.onerror = () => reject(req.error);
+      req.onsuccess = () => resolve(req.result);
+      req.onupgradeneeded = (e) => {
+        const db = (e.target as IDBOpenDBRequest).result;
+        if (!db.objectStoreNames.contains(this.STORE_NAME)) {
+          db.createObjectStore(this.STORE_NAME, { keyPath: "url" });
+        }
+      };
+    });
+  }
+
+  /**
+   * Clears all records from the phishing URLs store.
+   */
+  private clearStore(db: IDBDatabase): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const req = db.transaction(this.STORE_NAME, "readwrite").objectStore(this.STORE_NAME).clear();
+      req.onerror = () => reject(req.error);
+      req.onsuccess = () => resolve();
+    });
+  }
+
+  /**
+   * Saves an array of phishing URLs to IndexedDB.
+   * Atomically replaces all existing data.
+   *
+   * @param urls - Array of phishing URLs to save
+   * @returns `true` if save succeeded, `false` on error
+   */
+  async saveUrls(urls: string[]): Promise<boolean> {
+    let db: IDBDatabase | null = null;
+    try {
+      db = await this.openDatabase();
+      await this.clearStore(db);
+      await this.saveChunked(db, urls);
+      return true;
+    } catch (error) {
+      this.logService.error("[PhishingIndexedDbService] Save failed", error);
+      return false;
+    } finally {
+      db?.close();
+    }
+  }
+
+  /**
+   * Saves URLs in chunks to prevent transaction timeouts and UI freezes.
+   */
+  private async saveChunked(db: IDBDatabase, urls: string[]): Promise<void> {
+    const cleaned = urls.map((u) => u.trim()).filter(Boolean);
+    for (let i = 0; i < cleaned.length; i += this.CHUNK_SIZE) {
+      await this.saveChunk(db, cleaned.slice(i, i + this.CHUNK_SIZE));
+      await new Promise((r) => setTimeout(r, 0)); // Yield to event loop
+    }
+  }
+
+  /**
+   * Saves a single chunk of URLs in one transaction.
+   */
+  private saveChunk(db: IDBDatabase, urls: string[]): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const tx = db.transaction(this.STORE_NAME, "readwrite");
+      const store = tx.objectStore(this.STORE_NAME);
+      for (const url of urls) {
+        store.put({ url } as PhishingUrlRecord);
+      }
+      tx.oncomplete = () => resolve();
+      tx.onerror = () => reject(tx.error);
+    });
+  }
+
+  /**
+   * Checks if a URL exists in the phishing database.
+   *
+   * @param url - The URL to check
+   * @returns `true` if URL exists, `false` if not found or on error
+   */
+  async hasUrl(url: string): Promise<boolean> {
+    let db: IDBDatabase | null = null;
+    try {
+      db = await this.openDatabase();
+      return await this.checkUrlExists(db, url);
+    } catch (error) {
+      this.logService.error("[PhishingIndexedDbService] Check failed", error);
+      return false;
+    } finally {
+      db?.close();
+    }
+  }
+
+  /**
+   * Performs the actual URL existence check using index lookup.
+   */
+  private checkUrlExists(db: IDBDatabase, url: string): Promise<boolean> {
+    return new Promise((resolve, reject) => {
+      const tx = db.transaction(this.STORE_NAME, "readonly");
+      const req = tx.objectStore(this.STORE_NAME).get(url);
+      req.onerror = () => reject(req.error);
+      req.onsuccess = () => resolve(req.result !== undefined);
+    });
+  }
+
+  /**
+   * Loads all phishing URLs from IndexedDB.
+   *
+   * @returns Array of all stored URLs, or empty array on error
+   */
+  async loadAllUrls(): Promise<string[]> {
+    let db: IDBDatabase | null = null;
+    try {
+      db = await this.openDatabase();
+      return await this.getAllUrls(db);
+    } catch (error) {
+      this.logService.error("[PhishingIndexedDbService] Load failed", error);
+      return [];
+    } finally {
+      db?.close();
+    }
+  }
+
+  /**
+   * Iterates all records using a cursor.
+   */
+  private getAllUrls(db: IDBDatabase): Promise<string[]> {
+    return new Promise((resolve, reject) => {
+      const urls: string[] = [];
+      const req = db
+        .transaction(this.STORE_NAME, "readonly")
+        .objectStore(this.STORE_NAME)
+        .openCursor();
+      req.onerror = () => reject(req.error);
+      req.onsuccess = (e) => {
+        const cursor = (e.target as IDBRequest<IDBCursorWithValue | null>).result;
+        if (cursor) {
+          urls.push((cursor.value as PhishingUrlRecord).url);
+          cursor.continue();
+        } else {
+          resolve(urls);
+        }
+      };
+    });
+  }
+
+  /**
+   * Saves phishing URLs directly from a stream.
+   * Processes data incrementally to minimize memory usage.
+   *
+   * @param stream - ReadableStream of newline-delimited URLs
+   * @returns `true` if save succeeded, `false` on error
+   */
+  async saveUrlsFromStream(stream: ReadableStream<Uint8Array>): Promise<boolean> {
+    let db: IDBDatabase | null = null;
+    try {
+      db = await this.openDatabase();
+      await this.clearStore(db);
+      await this.processStream(db, stream);
+      return true;
+    } catch (error) {
+      this.logService.error("[PhishingIndexedDbService] Stream save failed", error);
+      return false;
+    } finally {
+      db?.close();
+    }
+  }
+
+  /**
+   * Processes a stream of URL data, parsing lines and saving in chunks.
+   */
+  private async processStream(db: IDBDatabase, stream: ReadableStream<Uint8Array>): Promise<void> {
+    const reader = stream.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    let urls: string[] = [];
+
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+
+        // Decode BEFORE done check; stream: !done flushes on final call
+        buffer += decoder.decode(value, { stream: !done });
+
+        if (done) {
+          // Split remaining buffer by newlines in case it contains multiple URLs
+          const lines = buffer.split("\n");
+          for (const line of lines) {
+            const trimmed = line.trim();
+            if (trimmed) {
+              urls.push(trimmed);
+            }
+          }
+          if (urls.length > 0) {
+            await this.saveChunk(db, urls);
+          }
+          break;
+        }
+
+        const lines = buffer.split("\n");
+        buffer = lines.pop() ?? "";
+
+        for (const line of lines) {
+          const trimmed = line.trim();
+          if (trimmed) {
+            urls.push(trimmed);
+          }
+
+          if (urls.length >= this.CHUNK_SIZE) {
+            await this.saveChunk(db, urls);
+            urls = [];
+            await new Promise((r) => setTimeout(r, 0));
+          }
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
+}