fetch from local db as fallback; only fetch new data on changed checksum; fetch from cdn

apps/browser/src/dirt/phishing-detection/services/phishing-detection.service.ts

@@ -24,6 +24,7 @@ import { TaskSchedulerService } from "@bitwarden/common/platform/scheduling/task
 import { BrowserApi } from "../../../platform/browser/browser-api";
 
 import {
+  CachedPhishingData,
   CaughtPhishingDomain,
   isPhishingDetectionMessage,
   PhishingDetectionMessage,
@@ -32,6 +33,17 @@ import {
 } from "./phishing-detection.types";
 
 export class PhishingDetectionService {
+  private static readonly RemotePhishingDatabaseUrl =
+    "https://raw.githubusercontent.com/Phishing-Database/Phishing.Database/master/phishing-domains-ACTIVE.txt";
+  private static readonly RemotePhishingDatabaseChecksumUrl =
+    "https://raw.githubusercontent.com/Phishing-Database/checksums/refs/heads/master/phishing-domains-ACTIVE.txt.md5";
+  private static readonly LocalPhishingDatabaseUrl = chrome.runtime.getURL(
+    "dirt/phishing-detection/services/phishing-domains-ACTIVE.txt",
+  );
+  /** This is tied to `./phishing-domainas-ACTIVE.txt` and should be updated in kind  */
+  private static readonly LocalPhishingDatabaseChecksum =
+    "ff5eb4352bd817baca18d2db6178b6e5 *phishing-domains-ACTIVE.txt";
+
   private static readonly _UPDATE_INTERVAL = 24 * 60 * 60 * 1000; // 24 hours in milliseconds
   private static readonly _RETRY_INTERVAL = 5 * 60 * 1000; // 5 minutes
   private static readonly _MAX_RETRIES = 3;
@@ -45,6 +57,8 @@ export class PhishingDetectionService {
   private static _navigationEventsSubject = new Subject<PhishingDetectionNavigationEvent>();
   private static _navigationEvents: Subscription | null = null;
   private static _knownPhishingDomains = new Set<string>();
+  private static _knownPhishingDomainsChecksum: string = "";
+  private static _attemptedLocalDBLoad = false;
   private static _caughtTabs: Map<PhishingDetectionTabId, CaughtPhishingDomain> = new Map();
   private static _isInitialized = false;
   private static _isUpdating = false;
@@ -538,14 +552,12 @@ export class PhishingDetectionService {
    */
   private static async _loadCachedDomains() {
     try {
-      const cachedData = await this._storageService.get<{ domains: string[]; timestamp: number }>(
-        this._STORAGE_KEY,
-      );
+      const cachedData = await this._storageService.get<CachedPhishingData>(this._STORAGE_KEY);
       if (cachedData) {
         this._logService.info("[PhishingDetectionService] Phishing cachedData exists");
         const phishingDomains = cachedData.domains || [];
 
-        this._setKnownPhishingDomains(phishingDomains);
+        this._setKnownPhishingDomains(phishingDomains, cachedData.checksum);
         this._handleTestUrls();
       }
 
@@ -570,8 +582,6 @@ export class PhishingDetectionService {
    * Updates the cache and handles retries if necessary
    */
   static async _fetchKnownPhishingDomains(): Promise<void> {
-    let domains: string[] = [];
-
     // Prevent concurrent updates
     if (this._isUpdating) {
       this._logService.warning(
@@ -583,10 +593,17 @@ export class PhishingDetectionService {
     try {
       this._logService.info("[PhishingDetectionService] Starting phishing domains update...");
       this._isUpdating = true;
-      domains = await this._auditService.getKnownPhishingDomains();
-      this._setKnownPhishingDomains(domains);
-
-      await this._saveDomains();
+      const res = await this._auditService.getKnownPhishingDomainsIfChanged(
+        this._knownPhishingDomainsChecksum,
+        this.RemotePhishingDatabaseChecksumUrl,
+        this.RemotePhishingDatabaseUrl,
+      );
+      if (!res) {
+        this._logService.info("[PhishingDetectionService] Domains are already up to date");
+      } else {
+        this._setKnownPhishingDomains(res.domains, res.checksum);
+        await this._saveDomains();
+      }
 
       this._resetRetry();
       this._isUpdating = false;
@@ -598,6 +615,9 @@ export class PhishingDetectionService {
         error,
       );
 
+      // Load local DB as fallback
+      await this._loadLocalPhishingDomainsDatabase();
+
       this._scheduleRetry();
       this._isUpdating = false;
 
@@ -605,6 +625,25 @@ export class PhishingDetectionService {
     }
   }
 
+  private static async _loadLocalPhishingDomainsDatabase() {
+    if (this._attemptedLocalDBLoad) {
+      return;
+    }
+
+    try {
+      this._logService.info("[PhishingDetectionService] Fetching local DB as fallback");
+      const fallbackDomains = await this._auditService.getKnownPhishingDomains(
+        this.LocalPhishingDatabaseUrl,
+      );
+      this._setKnownPhishingDomains(fallbackDomains, this.LocalPhishingDatabaseChecksum);
+      await this._saveDomains();
+    } catch (error) {
+      this._logService.error("[PhishingDetectionService] Failed to fetch local DB.", error);
+    }
+
+    this._attemptedLocalDBLoad = true;
+  }
+
   /**
    * Saves the known phishing domains to storage
    * Caches the updated domains and updates the last update time
@@ -612,9 +651,10 @@ export class PhishingDetectionService {
   private static async _saveDomains() {
     try {
       // Cache the updated domains
-      await this._storageService.save(this._STORAGE_KEY, {
+      await this._storageService.save<CachedPhishingData>(this._STORAGE_KEY, {
         domains: Array.from(this._knownPhishingDomains),
         timestamp: this._lastUpdateTime,
+        checksum: this._knownPhishingDomainsChecksum,
       });
       this._logService.info(
         `[PhishingDetectionService] Updated phishing domains cache with ${this._knownPhishingDomains.size} domains`,
@@ -650,11 +690,11 @@ export class PhishingDetectionService {
    *
    * @param domains Array of phishing domains to add
    */
-  private static _setKnownPhishingDomains(domains: string[]): void {
+  private static _setKnownPhishingDomains(domains: string[], checksum: string): void {
     this._logService.debug(
       `[PhishingDetectionService] Tracking ${domains.length} phishing domains`,
     );
-
+    this._knownPhishingDomainsChecksum = checksum;
     // Clear old domains to prevent memory leaks
     this._knownPhishingDomains.clear();
 

apps/browser/src/dirt/phishing-detection/services/phishing-detection.types.ts

@@ -33,3 +33,9 @@ export type PhishingDetectionNavigationEvent = {
   changeInfo: chrome.tabs.OnUpdatedInfo;
   tab: chrome.tabs.Tab;
 };
+
+export type CachedPhishingData = {
+  domains: string[];
+  timestamp: number;
+  checksum: string;
+};

libs/common/src/abstractions/audit.service.ts

@@ -14,10 +14,23 @@ export abstract class AuditService {
    * @returns A promise that resolves to an array of BreachAccountResponse objects.
    */
   abstract breachedAccounts: (username: string) => Promise<BreachAccountResponse[]>;
+
   /**
-   * Checks if a domain is known for phishing.
-   * @param domain The domain to check.
-   * @returns A promise that resolves to a boolean indicating if the domain is known for phishing.
+   * Retrieves the latest known phishing domains and their checksum if changed
+   * @param prevChecksum The previous checksum value to compare against.
+   * @param checksumUrl The URL to fetch the latest checksum.
+   * @param domainsUrl The URL to fetch the list of phishing domains.
+   * @returns A promise that resolves to an object containing the domains array and new checksum, or null if unchanged.
    */
-  abstract getKnownPhishingDomains: () => Promise<string[]>;
+  abstract getKnownPhishingDomainsIfChanged(
+    prevChecksum: string,
+    checksumUrl: string,
+    domainsUrl: string,
+  ): Promise<{ domains: string[]; checksum: string } | null>;
+
+  /**
+   * Retrieves the latest known phishing domains
+   * @param domainsUrl The URL to fetch the list of phishing domains.
+   */
+  abstract getKnownPhishingDomains(domainsUrl: string): Promise<string[]>;
 }

libs/common/src/services/audit.service.ts

@@ -81,8 +81,32 @@ export class AuditService implements AuditServiceAbstraction {
     }
   }
 
-  async getKnownPhishingDomains(): Promise<string[]> {
-    const response = await this.apiService.send("GET", "/phishing-domains", null, true, true);
-    return response as string[];
+  async getKnownPhishingDomainsIfChanged(
+    prevChecksum: string,
+    checksumUrl: string,
+    domainsUrl: string,
+  ) {
+    const checksum = await this.apiService
+      .nativeFetch(new Request(checksumUrl))
+      .then((res) => res.text());
+    if (prevChecksum === checksum) {
+      return null;
+    }
+
+    const domains = await this.getKnownPhishingDomains(domainsUrl);
+
+    return {
+      domains,
+      checksum,
+    };
+  }
+
+  async getKnownPhishingDomains(domainsUrl: string) {
+    const domains = await this.apiService
+      .nativeFetch(new Request(domainsUrl))
+      .then((res) => res.text())
+      .then((text) => text.split("\n"));
+
+    return domains;
   }
 }