add bash script to fetch local db info and split it to meet FF size limits

.github/whitelist-capital-letters.txt

@@ -19,7 +19,6 @@
 ./apps/cli/stores/chocolatey/tools/VERIFICATION.txt
 ./apps/browser/store/windows/AppxManifest.xml
 ./apps/browser/src/background/nativeMessaging.background.ts
-./apps/browser/src/dirt/phishing-detection/services/phishing-domains-ACTIVE.txt
 ./apps/browser/src/models/browserComponentState.ts
 ./apps/browser/src/models/browserGroupingsComponentState.ts
 ./apps/browser/src/models/biometricErrors.ts

apps/browser/src/dirt/phishing-detection/services/data/local-phishing-meta.json

@@ -0,0 +1,9 @@
+{
+  "files": [
+    "phishing-domains-part-aa.txt",
+    "phishing-domains-part-ab.txt",
+    "phishing-domains-part-ac.txt",
+    "phishing-domains-part-ad.txt"
+  ],
+  "checksum": "b7c4be474f9cfe053a1b306bf099ec95 *phishing-domains-ACTIVE.txt"
+}

apps/browser/src/dirt/phishing-detection/services/fetch-phishing-domains.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+DATA_DIR="$SCRIPT_DIR/data"
+
+mkdir -p "$DATA_DIR"
+
+# Delete existing files in the data directory
+rm -f "$DATA_DIR"/*
+
+# Fetch and split the phishing domains list into 4MB chunks with .txt extension, no intermediate file
+curl -L "https://raw.githubusercontent.com/Phishing-Database/Phishing.Database/master/phishing-domains-ACTIVE.txt" \
+  | gsplit -b 4m - "$DATA_DIR/phishing-domains-part-" --additional-suffix=.txt
+
+# Fetch the checksum
+CHECKSUM=$(curl -sL "https://raw.githubusercontent.com/Phishing-Database/checksums/refs/heads/master/phishing-domains-ACTIVE.txt.md5" | tr -d '\n')
+
+# Generate a JSON file including the checksum field, with paths relative to the data directory
+ls "$DATA_DIR"/phishing-domains-part-*.txt \
+  | sed "s|$DATA_DIR/||" \
+  | jq -R . | jq -s --arg checksum "$CHECKSUM" '{ files: ., checksum: $checksum }' \
+  > "$DATA_DIR/local-phishing-meta.json"

apps/browser/src/dirt/phishing-detection/services/phishing-detection.service.ts

@@ -23,6 +23,7 @@ import { TaskSchedulerService } from "@bitwarden/common/platform/scheduling/task
 
 import { BrowserApi } from "../../../platform/browser/browser-api";
 
+import LocalPhishingMeta from "./data/local-phishing-meta.json";
 import {
   CachedPhishingData,
   CaughtPhishingDomain,
@@ -37,12 +38,6 @@ export class PhishingDetectionService {
     "https://raw.githubusercontent.com/Phishing-Database/Phishing.Database/master/phishing-domains-ACTIVE.txt";
   private static readonly RemotePhishingDatabaseChecksumUrl =
     "https://raw.githubusercontent.com/Phishing-Database/checksums/refs/heads/master/phishing-domains-ACTIVE.txt.md5";
-  private static readonly LocalPhishingDatabaseUrl?: string = chrome?.runtime?.getURL(
-    "dirt/phishing-detection/services/phishing-domains-ACTIVE.txt",
-  );
-  /** This is tied to `./phishing-domainas-ACTIVE.txt` and should be updated in kind  */
-  private static readonly LocalPhishingDatabaseChecksum =
-    "ff5eb4352bd817baca18d2db6178b6e5 *phishing-domains-ACTIVE.txt";
 
   private static readonly _UPDATE_INTERVAL = 24 * 60 * 60 * 1000; // 24 hours in milliseconds
   private static readonly _RETRY_INTERVAL = 5 * 60 * 1000; // 5 minutes
@@ -632,13 +627,15 @@ export class PhishingDetectionService {
 
     try {
       this._logService.info("[PhishingDetectionService] Fetching local DB as fallback");
-      if (!this.LocalPhishingDatabaseUrl) {
-        throw new Error("No local database URL provided");
-      }
-      const fallbackDomains = await this._auditService.getKnownPhishingDomains(
-        this.LocalPhishingDatabaseUrl,
-      );
-      this._setKnownPhishingDomains(fallbackDomains, this.LocalPhishingDatabaseChecksum);
+      const fallbackDomains: string[] = await Promise.all(
+        LocalPhishingMeta.files.map((path) =>
+          this._auditService.getKnownPhishingDomains(
+            chrome.runtime.getURL("dirt/phishing-detection/services/data/" + path),
+          ),
+        ),
+      ).then((chunkedDomains) => chunkedDomains.flat());
+
+      this._setKnownPhishingDomains(fallbackDomains, LocalPhishingMeta.checksum);
       await this._saveDomains();
     } catch (error) {
       this._logService.error("[PhishingDetectionService] Failed to fetch local DB.", error);

apps/browser/src/manifest.json

@@ -137,7 +137,7 @@
     "overlay/menu-list.html",
     "overlay/menu.html",
     "popup/fonts/*",
-    "dirt/phishing-detection/services/phishing-domains-ACTIVE.txt"
+    "dirt/phishing-detection/services/data/*"
   ],
   "__firefox__browser_specific_settings": {
     "gecko": {

apps/browser/src/manifest.v3.json

@@ -163,7 +163,7 @@
         "overlay/menu-list.html",
         "overlay/menu.html",
         "popup/fonts/*",
-        "dirt/phishing-detection/services/phishing-domains-ACTIVE.txt"
+        "dirt/phishing-detection/services/data/*"
       ],
       "matches": ["<all_urls>"]
     }

apps/browser/webpack.base.js

@@ -186,11 +186,8 @@ module.exports.buildConfig = function buildConfig(params) {
         { from: path.resolve(__dirname, "src/popup/images"), to: "popup/images" },
         { from: path.resolve(__dirname, "src/autofill/content/autofill.css"), to: "content" },
         {
-          from: path.resolve(
-            __dirname,
-            "src/dirt/phishing-detection/services/phishing-domains-ACTIVE.txt",
-          ),
-          to: "dirt/phishing-detection/services/phishing-domains-ACTIVE.txt",
+          from: path.resolve(__dirname, "src/dirt/phishing-detection/services/data"),
+          to: "dirt/phishing-detection/services/data",
         },
       ],
     }),