mirror of
https://github.com/bitwarden/browser
synced 2026-02-11 14:04:03 +00:00
Convert background update to rxjs format and trigger via subject. Update test cases
This commit is contained in:
Leslie Tilton
@@ -1,4 +1,5 @@
|
||||
import { MockProxy, mock } from "jest-mock-extended";
|
||||
import { firstValueFrom } from "rxjs";
|
||||
|
||||
import { ApiService } from "@bitwarden/common/abstractions/api.service";
|
||||
import { PlatformUtilsService } from "@bitwarden/common/platform/abstractions/platform-utils.service";
|
||||
@@ -9,66 +10,8 @@ import {
|
||||
import { FakeGlobalStateProvider } from "@bitwarden/common/spec";
|
||||
import { LogService } from "@bitwarden/logging";
|
||||
|
||||
import {
|
||||
PhishingDataService,
|
||||
PHISHING_DOMAINS_META_KEY,
|
||||
PHISHING_DOMAINS_BLOB_KEY,
|
||||
PhishingDataMeta,
|
||||
PhishingDataBlob,
|
||||
} from "./phishing-data.service";
|
||||
|
||||
const flushPromises = () =>
|
||||
new Promise((resolve) => jest.requireActual("timers").setImmediate(resolve));
|
||||
|
||||
// [FIXME] Move mocking and compression helpers to a shared test utils library
|
||||
// to separate from phishing data service tests.
|
||||
export const setupPhishingMocks = (mockedResult: string | ArrayBuffer = "mocked-data") => {
|
||||
// Store original globals
|
||||
const originals = {
|
||||
Response: global.Response,
|
||||
CompressionStream: global.CompressionStream,
|
||||
DecompressionStream: global.DecompressionStream,
|
||||
Blob: global.Blob,
|
||||
atob: global.atob,
|
||||
btoa: global.btoa,
|
||||
};
|
||||
|
||||
// Mock missing or browser-only globals
|
||||
global.atob = (str) => Buffer.from(str, "base64").toString("binary");
|
||||
global.btoa = (str) => Buffer.from(str, "binary").toString("base64");
|
||||
|
||||
(global as any).CompressionStream = class {};
|
||||
(global as any).DecompressionStream = class {};
|
||||
|
||||
global.Blob = class {
|
||||
constructor(public parts: any[]) {}
|
||||
stream() {
|
||||
return { pipeThrough: () => ({}) };
|
||||
}
|
||||
} as any;
|
||||
|
||||
global.Response = class {
|
||||
body = { pipeThrough: () => ({}) };
|
||||
// Return string for decompression
|
||||
text() {
|
||||
return Promise.resolve(typeof mockedResult === "string" ? mockedResult : "");
|
||||
}
|
||||
// Return ArrayBuffer for compression
|
||||
arrayBuffer() {
|
||||
if (typeof mockedResult === "string") {
|
||||
const bytes = new TextEncoder().encode(mockedResult);
|
||||
return Promise.resolve(bytes.buffer);
|
||||
}
|
||||
|
||||
return Promise.resolve(mockedResult);
|
||||
}
|
||||
} as any;
|
||||
|
||||
// Cleanup function
|
||||
return () => {
|
||||
Object.assign(global, originals);
|
||||
};
|
||||
};
|
||||
import { PhishingDataService } from "./phishing-data.service";
|
||||
import type { PhishingIndexedDbService } from "./phishing-indexeddb.service";
|
||||
|
||||
describe("PhishingDataService", () => {
|
||||
let service: PhishingDataService;
|
||||
@@ -76,33 +19,29 @@ describe("PhishingDataService", () => {
|
||||
let taskSchedulerService: TaskSchedulerService;
|
||||
let logService: MockProxy<LogService>;
|
||||
let platformUtilsService: MockProxy<PlatformUtilsService>;
|
||||
let mockIndexedDbService: MockProxy<PhishingIndexedDbService>;
|
||||
const fakeGlobalStateProvider: FakeGlobalStateProvider = new FakeGlobalStateProvider();
|
||||
|
||||
const setMockMeta = (state: PhishingDataMeta) => {
|
||||
fakeGlobalStateProvider.getFake(PHISHING_DOMAINS_META_KEY).stateSubject.next(state);
|
||||
return state;
|
||||
};
|
||||
const setMockBlob = (state: PhishingDataBlob) => {
|
||||
fakeGlobalStateProvider.getFake(PHISHING_DOMAINS_BLOB_KEY).stateSubject.next(state);
|
||||
return state;
|
||||
};
|
||||
|
||||
let fetchChecksumSpy: jest.SpyInstance;
|
||||
let fetchAndCompressSpy: jest.SpyInstance;
|
||||
|
||||
const mockMeta: PhishingDataMeta = {
|
||||
checksum: "abc",
|
||||
timestamp: Date.now(),
|
||||
applicationVersion: "1.0.0",
|
||||
};
|
||||
const mockBlob = "http://phish.com\nhttps://badguy.net";
|
||||
const mockCompressedBlob =
|
||||
"H4sIAAAAAAAA/8vMTSzJzM9TSE7MLchJLElVyE9TyC9KSS1S0FFIz8hLz0ksSQUAtK7XMSYAAAA=";
|
||||
|
||||
beforeEach(async () => {
|
||||
jest.useFakeTimers();
|
||||
jest.clearAllMocks();
|
||||
|
||||
// Mock Request global if not available
|
||||
if (typeof Request === "undefined") {
|
||||
(global as any).Request = class {
|
||||
constructor(public url: string) {}
|
||||
};
|
||||
}
|
||||
|
||||
apiService = mock<ApiService>();
|
||||
logService = mock<LogService>();
|
||||
mockIndexedDbService = mock<PhishingIndexedDbService>();
|
||||
|
||||
// Set default mock behaviors
|
||||
mockIndexedDbService.hasUrl.mockResolvedValue(false);
|
||||
mockIndexedDbService.loadAllUrls.mockResolvedValue([]);
|
||||
mockIndexedDbService.saveUrls.mockResolvedValue(undefined);
|
||||
mockIndexedDbService.saveUrlsFromStream.mockResolvedValue(undefined);
|
||||
|
||||
platformUtilsService = mock<PlatformUtilsService>();
|
||||
platformUtilsService.getApplicationVersion.mockResolvedValue("1.0.0");
|
||||
@@ -116,214 +55,116 @@ describe("PhishingDataService", () => {
|
||||
logService,
|
||||
platformUtilsService,
|
||||
);
|
||||
fetchChecksumSpy = jest.spyOn(service as any, "fetchPhishingChecksum");
|
||||
fetchAndCompressSpy = jest.spyOn(service as any, "fetchAndCompress");
|
||||
|
||||
// Replace the IndexedDB service with our mock
|
||||
service["indexedDbService"] = mockIndexedDbService;
|
||||
|
||||
fetchChecksumSpy = jest.spyOn(service as any, "fetchPhishingChecksum");
|
||||
fetchChecksumSpy.mockResolvedValue("new-checksum");
|
||||
fetchAndCompressSpy.mockResolvedValue("compressed-blob");
|
||||
});
|
||||
|
||||
describe("initialization", () => {
|
||||
beforeEach(() => {
|
||||
jest.spyOn(service as any, "_compressString").mockResolvedValue(mockCompressedBlob);
|
||||
jest.spyOn(service as any, "_decompressString").mockResolvedValue(mockBlob);
|
||||
it("should initialize with IndexedDB service", () => {
|
||||
expect(service["indexedDbService"]).toBeDefined();
|
||||
});
|
||||
|
||||
it("should perform background update", async () => {
|
||||
platformUtilsService.getApplicationVersion.mockResolvedValue("1.0.x");
|
||||
jest
|
||||
.spyOn(service as any, "getNextWebAddresses")
|
||||
.mockResolvedValue({ meta: mockMeta, blob: mockBlob });
|
||||
|
||||
setMockBlob(mockBlob);
|
||||
setMockMeta(mockMeta);
|
||||
|
||||
const sub = service.update$.subscribe();
|
||||
await flushPromises();
|
||||
|
||||
const url = new URL("http://phish.com");
|
||||
it("should detect QA test addresses", async () => {
|
||||
// The QA test address should always return true
|
||||
const QAurl = new URL("http://phishing.testcategory.com");
|
||||
expect(await service.isPhishingWebAddress(url)).toBe(true);
|
||||
expect(await service.isPhishingWebAddress(QAurl)).toBe(true);
|
||||
|
||||
sub.unsubscribe();
|
||||
// IndexedDB should not be called for test addresses
|
||||
expect(mockIndexedDbService.hasUrl).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("isPhishingWebAddress", () => {
|
||||
beforeEach(() => {
|
||||
jest.spyOn(service as any, "_compressString").mockResolvedValue(mockCompressedBlob);
|
||||
jest.spyOn(service as any, "_decompressString").mockResolvedValue(mockBlob);
|
||||
});
|
||||
|
||||
it("should detect a phishing web address", async () => {
|
||||
service["_webAddressesSet"] = new Set(["phish.com", "badguy.net"]);
|
||||
// Mock loadAllUrls to return entries with phishing URLs
|
||||
mockIndexedDbService.loadAllUrls.mockResolvedValue(["http://phish.com", "http://badguy.net"]);
|
||||
|
||||
const url = new URL("http://phish.com");
|
||||
const result = await service.isPhishingWebAddress(url);
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(mockIndexedDbService.loadAllUrls).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should not detect a safe web address", async () => {
|
||||
service["_webAddressesSet"] = new Set(["phish.com", "badguy.net"]);
|
||||
// Mock loadAllUrls to return phishing URLs that don't match
|
||||
mockIndexedDbService.loadAllUrls.mockResolvedValue(["http://phish.com", "http://badguy.net"]);
|
||||
|
||||
const url = new URL("http://safe.com");
|
||||
const result = await service.isPhishingWebAddress(url);
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(mockIndexedDbService.loadAllUrls).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should match against root web address", async () => {
|
||||
service["_webAddressesSet"] = new Set(["phish.com", "badguy.net"]);
|
||||
const url = new URL("http://phish.com/about");
|
||||
it("should match against root web address with subpaths", async () => {
|
||||
// Mock loadAllUrls to return entry that matches
|
||||
mockIndexedDbService.loadAllUrls.mockResolvedValue(["http://phish.com/login"]);
|
||||
|
||||
const url = new URL("http://phish.com/login/page");
|
||||
const result = await service.isPhishingWebAddress(url);
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(mockIndexedDbService.loadAllUrls).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should not error on empty state", async () => {
|
||||
service["_webAddressesSet"] = null;
|
||||
it("should handle IndexedDB errors gracefully", async () => {
|
||||
mockIndexedDbService.loadAllUrls.mockRejectedValue(new Error("IndexedDB error"));
|
||||
|
||||
const url = new URL("http://phish.com/about");
|
||||
const result = await service.isPhishingWebAddress(url);
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(logService.error).toHaveBeenCalledWith(
|
||||
"[PhishingDataService] Error running custom matcher",
|
||||
expect.any(Error),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getNextWebAddresses", () => {
|
||||
beforeEach(() => {
|
||||
jest.spyOn(service as any, "_compressString").mockResolvedValue(mockCompressedBlob);
|
||||
jest.spyOn(service as any, "_decompressString").mockResolvedValue(mockBlob);
|
||||
describe("data updates", () => {
|
||||
it("should update full dataset via stream", async () => {
|
||||
// Mock full dataset update
|
||||
const mockResponse = {
|
||||
ok: true,
|
||||
body: {} as ReadableStream,
|
||||
} as Response;
|
||||
apiService.nativeFetch.mockResolvedValue(mockResponse);
|
||||
|
||||
await firstValueFrom(service["_updateFullDataSet"]());
|
||||
|
||||
expect(mockIndexedDbService.saveUrlsFromStream).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refetches all web addresses if applicationVersion has changed", async () => {
|
||||
const prev: PhishingDataMeta = {
|
||||
timestamp: Date.now() - 60000,
|
||||
checksum: "old",
|
||||
applicationVersion: "1.0.0",
|
||||
};
|
||||
fetchChecksumSpy.mockResolvedValue("new");
|
||||
it("should update daily dataset via saveUrls", async () => {
|
||||
// Mock daily update
|
||||
const mockResponse = {
|
||||
ok: true,
|
||||
text: jest.fn().mockResolvedValue("newphish.com\nanotherbad.net"),
|
||||
} as unknown as Response;
|
||||
apiService.nativeFetch.mockResolvedValue(mockResponse);
|
||||
|
||||
await firstValueFrom(service["_updateDailyDataSet"]());
|
||||
|
||||
expect(mockIndexedDbService.saveUrls).toHaveBeenCalledWith([
|
||||
"newphish.com",
|
||||
"anotherbad.net",
|
||||
]);
|
||||
});
|
||||
|
||||
it("should get updated meta information", async () => {
|
||||
fetchChecksumSpy.mockResolvedValue("new-checksum");
|
||||
platformUtilsService.getApplicationVersion.mockResolvedValue("2.0.0");
|
||||
|
||||
const result = await service.getNextWebAddresses(prev);
|
||||
const meta = await firstValueFrom(service["_getUpdatedMeta"]());
|
||||
|
||||
expect(result!.blob).toBe("compressed-blob");
|
||||
expect(result!.meta!.checksum).toBe("new");
|
||||
expect(result!.meta!.applicationVersion).toBe("2.0.0");
|
||||
});
|
||||
|
||||
it("returns null when checksum matches and cache not expired", async () => {
|
||||
const prev: PhishingDataMeta = {
|
||||
timestamp: Date.now(),
|
||||
checksum: "abc",
|
||||
applicationVersion: "1.0.0",
|
||||
};
|
||||
fetchChecksumSpy.mockResolvedValue("abc");
|
||||
const result = await service.getNextWebAddresses(prev);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("patches daily domains when cache is expired and checksum unchanged", async () => {
|
||||
const prev: PhishingDataMeta = {
|
||||
timestamp: 0,
|
||||
checksum: "old",
|
||||
applicationVersion: "1.0.0",
|
||||
};
|
||||
const dailyLines = ["b.com", "c.com"];
|
||||
fetchChecksumSpy.mockResolvedValue("old");
|
||||
jest.spyOn(service as any, "fetchText").mockResolvedValue(dailyLines);
|
||||
|
||||
setMockBlob(mockBlob);
|
||||
|
||||
const expectedBlob =
|
||||
"H4sIAAAAAAAA/8vMTSzJzM9TSE7MLchJLElVyE9TyC9KSS1S0FFIz8hLz0ksSQUAtK7XMSYAAAA=";
|
||||
const result = await service.getNextWebAddresses(prev);
|
||||
|
||||
expect(result!.blob).toBe(expectedBlob);
|
||||
expect(result!.meta!.checksum).toBe("old");
|
||||
});
|
||||
|
||||
it("fetches all domains when checksum has changed", async () => {
|
||||
const prev: PhishingDataMeta = {
|
||||
timestamp: 0,
|
||||
checksum: "old",
|
||||
applicationVersion: "1.0.0",
|
||||
};
|
||||
fetchChecksumSpy.mockResolvedValue("new");
|
||||
fetchAndCompressSpy.mockResolvedValue("new-blob");
|
||||
const result = await service.getNextWebAddresses(prev);
|
||||
expect(result!.blob).toBe("new-blob");
|
||||
expect(result!.meta!.checksum).toBe("new");
|
||||
});
|
||||
});
|
||||
|
||||
describe("compression helpers", () => {
|
||||
let restore: () => void;
|
||||
|
||||
beforeEach(async () => {
|
||||
restore = setupPhishingMocks("abc");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (restore) {
|
||||
restore();
|
||||
}
|
||||
delete (Uint8Array as any).fromBase64;
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("_compressString", () => {
|
||||
it("compresses a string to base64", async () => {
|
||||
const out = await service["_compressString"]("abc");
|
||||
expect(out).toBe("YWJj"); // base64 for 'abc'
|
||||
});
|
||||
|
||||
it("compresses using fallback on older browsers", async () => {
|
||||
const input = "abc";
|
||||
const expected = btoa(encodeURIComponent(input));
|
||||
const out = await service["_compressString"](input);
|
||||
expect(out).toBe(expected);
|
||||
});
|
||||
|
||||
it("compresses using btoa on error", async () => {
|
||||
const input = "abc";
|
||||
const expected = btoa(encodeURIComponent(input));
|
||||
const out = await service["_compressString"](input);
|
||||
expect(out).toBe(expected);
|
||||
});
|
||||
});
|
||||
describe("_decompressString", () => {
|
||||
it("decompresses a string from base64", async () => {
|
||||
const base64 = btoa("ignored");
|
||||
const out = await service["_decompressString"](base64);
|
||||
expect(out).toBe("abc");
|
||||
});
|
||||
|
||||
it("decompresses using fallback on older browsers", async () => {
|
||||
// Provide a fromBase64 implementation
|
||||
(Uint8Array as any).fromBase64 = (b64: string) => new Uint8Array([100, 101, 102]);
|
||||
|
||||
const out = await service["_decompressString"]("ignored");
|
||||
expect(out).toBe("abc");
|
||||
});
|
||||
|
||||
it("decompresses using atob on error", async () => {
|
||||
const base64 = btoa(encodeURIComponent("abc"));
|
||||
const out = await service["_decompressString"](base64);
|
||||
expect(out).toBe("abc");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("_loadBlobToMemory", () => {
|
||||
it("loads blob into memory set", async () => {
|
||||
const prevBlob = "ignored-base64";
|
||||
fakeGlobalStateProvider.getFake(PHISHING_DOMAINS_BLOB_KEY).stateSubject.next(prevBlob);
|
||||
|
||||
jest.spyOn(service as any, "_decompressString").mockResolvedValue("phish.com\nbadguy.net");
|
||||
|
||||
await service["_loadBlobToMemory"]();
|
||||
const set = service["_webAddressesSet"] as Set<string>;
|
||||
expect(set).toBeDefined();
|
||||
expect(set.has("phish.com")).toBe(true);
|
||||
expect(set.has("badguy.net")).toBe(true);
|
||||
expect(meta).toBeDefined();
|
||||
expect(meta.checksum).toBe("new-checksum");
|
||||
expect(meta.applicationVersion).toBe("2.0.0");
|
||||
expect(meta.timestamp).toBeDefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,25 @@
|
||||
import { catchError, EMPTY, first, share, startWith, Subject, switchMap, tap } from "rxjs";
|
||||
import {
|
||||
catchError,
|
||||
concatMap,
|
||||
defer,
|
||||
EMPTY,
|
||||
exhaustMap,
|
||||
first,
|
||||
forkJoin,
|
||||
from,
|
||||
iif,
|
||||
map,
|
||||
Observable,
|
||||
of,
|
||||
retry,
|
||||
share,
|
||||
startWith,
|
||||
Subject,
|
||||
switchMap,
|
||||
tap,
|
||||
throwError,
|
||||
timer,
|
||||
} from "rxjs";
|
||||
|
||||
import { devFlagEnabled, devFlagValue } from "@bitwarden/browser/platform/flags";
|
||||
import { ApiService } from "@bitwarden/common/abstractions/api.service";
|
||||
@@ -64,6 +85,8 @@ export class PhishingDataService {
|
||||
// How often are new web addresses added to the remote?
|
||||
readonly UPDATE_INTERVAL_DURATION = 24 * 60 * 60 * 1000; // 24 hours
|
||||
|
||||
private _backgroundUpdateTrigger$ = new Subject<PhishingDataMeta | null>();
|
||||
|
||||
private _triggerUpdate$ = new Subject<void>();
|
||||
update$ = this._triggerUpdate$.pipe(
|
||||
startWith(undefined), // Always emit once
|
||||
@@ -71,8 +94,8 @@ export class PhishingDataService {
|
||||
this._phishingMetaState.state$.pipe(
|
||||
first(), // Only take the first value to avoid an infinite loop when updating the cache below
|
||||
tap((metaState) => {
|
||||
// Perform any updates in the background if needed
|
||||
void this._backgroundUpdate(metaState);
|
||||
// Perform any updates in the background
|
||||
this._backgroundUpdateTrigger$.next(metaState);
|
||||
}),
|
||||
catchError((err: unknown) => {
|
||||
this.logService.error("[PhishingDataService] Background update failed to start.", err);
|
||||
@@ -100,6 +123,13 @@ export class PhishingDataService {
|
||||
ScheduledTaskNames.phishingDomainUpdate,
|
||||
this.UPDATE_INTERVAL_DURATION,
|
||||
);
|
||||
this._backgroundUpdateTrigger$
|
||||
.pipe(
|
||||
exhaustMap((currentMeta) => {
|
||||
return this._backgroundUpdate(currentMeta);
|
||||
}),
|
||||
)
|
||||
.subscribe();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -142,63 +172,6 @@ export class PhishingDataService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines the next set of web addresses to fetch based on previous metadata.
|
||||
*
|
||||
* @param previous Previous phishing data metadata
|
||||
* @returns Object containing either a stream for full update or lines for daily update, along with new metadata; or null if no update is needed
|
||||
*/
|
||||
async getNextWebAddresses(previous: PhishingDataMeta | null): Promise<{
|
||||
meta?: PhishingDataMeta;
|
||||
stream?: ReadableStream<Uint8Array>;
|
||||
lines?: string[];
|
||||
} | null> {
|
||||
const prevMeta = previous ?? { timestamp: 0, checksum: "", applicationVersion: "" };
|
||||
const now = Date.now();
|
||||
|
||||
// Updates to check
|
||||
const applicationVersion = await this.platformUtilsService.getApplicationVersion();
|
||||
const remoteChecksum = await this.fetchPhishingChecksum(this.resourceType);
|
||||
|
||||
// Logic checks
|
||||
const appVersionChanged = applicationVersion !== prevMeta.applicationVersion;
|
||||
const masterChecksumChanged = remoteChecksum !== prevMeta.checksum;
|
||||
|
||||
// Full update: return stream so caller can write to IndexedDB incrementally
|
||||
if (masterChecksumChanged || appVersionChanged) {
|
||||
this.logService.info("[PhishingDataService] Checksum or version changed; Fetching ALL.");
|
||||
const remoteUrl = getPhishingResources(this.resourceType)!.remoteUrl;
|
||||
const response = await this.apiService.nativeFetch(new Request(remoteUrl));
|
||||
if (!response.ok || !response.body) {
|
||||
throw new Error("Fetch failed");
|
||||
}
|
||||
return {
|
||||
stream: response.body!,
|
||||
meta: { checksum: remoteChecksum, timestamp: now, applicationVersion },
|
||||
};
|
||||
}
|
||||
|
||||
// Check for daily file
|
||||
const isCacheExpired = now - prevMeta.timestamp > this.UPDATE_INTERVAL_DURATION;
|
||||
|
||||
if (isCacheExpired) {
|
||||
this.logService.info("[PhishingDataService] Daily cache expired; Fetching TODAY's");
|
||||
const url = getPhishingResources(this.resourceType)!.todayUrl;
|
||||
const newLines = await this.fetchToday(url);
|
||||
|
||||
return {
|
||||
lines: newLines,
|
||||
meta: {
|
||||
checksum: remoteChecksum,
|
||||
timestamp: now,
|
||||
applicationVersion,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// [FIXME] Pull fetches into api service
|
||||
private async fetchPhishingChecksum(type: PhishingResourceType = PhishingResourceType.Domains) {
|
||||
const checksumUrl = getPhishingResources(type)!.checksumUrl;
|
||||
@@ -237,69 +210,121 @@ export class PhishingDataService {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Runs the update flow in the background and retries up to 3 times on failure
|
||||
private async _backgroundUpdate(previous: PhishingDataMeta | null): Promise<void> {
|
||||
this.logService.info(`[PhishingDataService] Update web addresses triggered...`);
|
||||
const phishingMeta: PhishingDataMeta = previous ?? {
|
||||
timestamp: 0,
|
||||
checksum: "",
|
||||
applicationVersion: "",
|
||||
};
|
||||
// Start time for logging performance of update
|
||||
const startTime = Date.now();
|
||||
const maxAttempts = 3;
|
||||
const delayMs = 5 * 60 * 1000; // 5 minutes
|
||||
private _getUpdatedMeta(): Observable<PhishingDataMeta> {
|
||||
// Use defer to
|
||||
return defer(() => {
|
||||
const now = Date.now();
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
const next = await this.getNextWebAddresses(phishingMeta);
|
||||
if (!next) {
|
||||
return; // No update needed
|
||||
// forkJoin kicks off both requests in parallel
|
||||
return forkJoin({
|
||||
applicationVersion: from(this.platformUtilsService.getApplicationVersion()),
|
||||
remoteChecksum: from(this.fetchPhishingChecksum(this.resourceType)),
|
||||
}).pipe(
|
||||
map(({ applicationVersion, remoteChecksum }) => {
|
||||
return {
|
||||
checksum: remoteChecksum,
|
||||
timestamp: now,
|
||||
applicationVersion,
|
||||
};
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
// Streams the full phishing data set and saves it to IndexedDB
|
||||
private _updateFullDataSet() {
|
||||
this.logService.info("[PhishingDataService] Starting FULL update...");
|
||||
const resource = getPhishingResources(this.resourceType);
|
||||
|
||||
if (!resource?.remoteUrl) {
|
||||
return throwError(() => new Error("Invalid resource URL"));
|
||||
}
|
||||
|
||||
return from(this.apiService.nativeFetch(new Request(resource.remoteUrl))).pipe(
|
||||
switchMap((response) => {
|
||||
if (!response.ok || !response.body) {
|
||||
return throwError(() => new Error(`Full fetch failed: ${response.statusText}`));
|
||||
}
|
||||
|
||||
if (next.meta) {
|
||||
await this._phishingMetaState.update(() => next.meta!);
|
||||
}
|
||||
return from(this.indexedDbService.saveUrlsFromStream(response.body));
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// If we received a stream, write it into IndexedDB incrementally
|
||||
if (next.stream) {
|
||||
await this.indexedDbService.saveUrlsFromStream(next.stream);
|
||||
} else if (next.lines) {
|
||||
await this.indexedDbService.saveUrls(next.lines);
|
||||
private _updateDailyDataSet() {
|
||||
this.logService.info("[PhishingDataService] Starting DAILY update...");
|
||||
|
||||
// TODO Check if any of this is needed or if saveUrls is sufficient
|
||||
// AI Updates but does not take into account that saving
|
||||
// to the indexedDB will merge entries and not create duplicates.
|
||||
// // For incremental daily updates we merge with existing set to preserve old entries
|
||||
// const existing = await this.indexedDbService.loadAllUrls();
|
||||
// const combinedSet = new Set<string>(existing);
|
||||
// for (const l of next.lines) {
|
||||
// const trimmed = l.trim();
|
||||
// if (trimmed) {
|
||||
// combinedSet.add(trimmed);
|
||||
// }
|
||||
// }
|
||||
// await this.indexedDbService.saveUrls(Array.from(combinedSet));
|
||||
}
|
||||
const todayUrl = getPhishingResources(this.resourceType)?.todayUrl;
|
||||
if (!todayUrl) {
|
||||
return throwError(() => new Error("Today URL missing"));
|
||||
}
|
||||
|
||||
const elapsed = Date.now() - startTime;
|
||||
this.logService.info(`[PhishingDataService] Phishing data cache updated in ${elapsed}ms`);
|
||||
return;
|
||||
} catch (err) {
|
||||
this.logService.error(
|
||||
`[PhishingDataService] Unable to update web addresses. Attempt ${attempt}.`,
|
||||
err,
|
||||
);
|
||||
if (attempt < maxAttempts) {
|
||||
await new Promise((res) => setTimeout(res, delayMs));
|
||||
} else {
|
||||
return from(this.fetchToday(todayUrl)).pipe(
|
||||
switchMap((lines) => from(this.indexedDbService.saveUrls(lines))),
|
||||
);
|
||||
}
|
||||
|
||||
private _backgroundUpdate(
|
||||
previous: PhishingDataMeta | null,
|
||||
): Observable<PhishingDataMeta | null> {
|
||||
// Use defer to restart timer if retry is activated
|
||||
return defer(() => {
|
||||
const startTime = Date.now();
|
||||
this.logService.info(`[PhishingDataService] Update triggered...`);
|
||||
|
||||
// Get updated meta info
|
||||
return this._getUpdatedMeta().pipe(
|
||||
// Update full data set if application version or checksum changed
|
||||
concatMap((newMeta) =>
|
||||
iif(
|
||||
() => {
|
||||
const appVersionChanged = newMeta.applicationVersion !== previous?.applicationVersion;
|
||||
const checksumChanged = newMeta.checksum !== previous?.checksum;
|
||||
return appVersionChanged || checksumChanged;
|
||||
},
|
||||
this._updateFullDataSet().pipe(map(() => newMeta)),
|
||||
of(newMeta),
|
||||
),
|
||||
),
|
||||
// Update daily data set if last update was more than UPDATE_INTERVAL_DURATION ago
|
||||
concatMap((newMeta) =>
|
||||
iif(
|
||||
() => {
|
||||
const isCacheExpired =
|
||||
Date.now() - (previous?.timestamp ?? 0) > this.UPDATE_INTERVAL_DURATION;
|
||||
return isCacheExpired;
|
||||
},
|
||||
this._updateDailyDataSet().pipe(map(() => newMeta)),
|
||||
of(newMeta),
|
||||
),
|
||||
),
|
||||
concatMap((newMeta) => {
|
||||
return from(this._phishingMetaState.update(() => newMeta)).pipe(
|
||||
tap(() => {
|
||||
const elapsed = Date.now() - startTime;
|
||||
this.logService.info(`[PhishingDataService] Updated in ${elapsed}ms`);
|
||||
}),
|
||||
);
|
||||
}),
|
||||
retry({
|
||||
count: 2, // Total 3 attempts (initial + 2 retries)
|
||||
delay: (error, retryCount) => {
|
||||
this.logService.error(
|
||||
`[PhishingDataService] Attempt ${retryCount} failed. Retrying in 5m...`,
|
||||
error,
|
||||
);
|
||||
return timer(5 * 60 * 1000); // Wait 5 mins before next attempt
|
||||
},
|
||||
}),
|
||||
catchError((err: unknown) => {
|
||||
const elapsed = Date.now() - startTime;
|
||||
this.logService.error(
|
||||
`[PhishingDataService] Retries unsuccessful after ${elapsed}ms. Unable to update web addresses.`,
|
||||
`[PhishingDataService] Retries unsuccessful after ${elapsed}ms.`,
|
||||
err,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return throwError(() => err);
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user