From 64bee4c91d90036319b44e0321d3b9b9b521c01a Mon Sep 17 00:00:00 2001 From: John Harrington <84741727+harr1424@users.noreply.github.com> Date: Sat, 16 Aug 2025 07:26:00 -0700 Subject: [PATCH] feature: de-duplication tool with test coverage, i18n, and UI guidelines --- .../app/layouts/user-layout.component.html | 1 + apps/web/src/app/oss-routing.module.ts | 8 + .../de-duplicate/de-duplicate.component.html | 28 + .../de-duplicate/de-duplicate.component.ts | 71 +++ .../duplicate-review-dialog.component.html | 127 ++++ .../duplicate-review-dialog.component.ts | 160 +++++ .../services/de-duplicate.service.spec.ts | 559 ++++++++++++++++++ .../vault/services/de-duplicate.service.ts | 363 ++++++++++++ apps/web/src/locales/en/messages.json | 60 ++ 9 files changed, 1377 insertions(+) create mode 100644 apps/web/src/app/tools/de-duplicate/de-duplicate.component.html create mode 100644 apps/web/src/app/tools/de-duplicate/de-duplicate.component.ts create mode 100644 apps/web/src/app/tools/de-duplicate/duplicate-review-dialog.component.html create mode 100644 apps/web/src/app/tools/de-duplicate/duplicate-review-dialog.component.ts create mode 100644 apps/web/src/app/vault/services/de-duplicate.service.spec.ts create mode 100644 apps/web/src/app/vault/services/de-duplicate.service.ts diff --git a/apps/web/src/app/layouts/user-layout.component.html b/apps/web/src/app/layouts/user-layout.component.html index 530d4caca03..cee1f8314ac 100644 --- a/apps/web/src/app/layouts/user-layout.component.html +++ b/apps/web/src/app/layouts/user-layout.component.html @@ -8,6 +8,7 @@ + diff --git a/apps/web/src/app/oss-routing.module.ts b/apps/web/src/app/oss-routing.module.ts index 4c1ed1a7472..10b18798c60 100644 --- a/apps/web/src/app/oss-routing.module.ts +++ b/apps/web/src/app/oss-routing.module.ts @@ -673,6 +673,14 @@ const routes: Routes = [ titleId: "exportVault", } satisfies RouteDataProperties, }, + { + path: "de-duplicate", + loadComponent: () => + import("./tools/de-duplicate/de-duplicate.component").then( + (m) => m.DeDuplicateComponent, + ), + data: { titleId: "duplicateItemsFound" } satisfies RouteDataProperties, + }, { path: "generator", component: CredentialGeneratorComponent, diff --git a/apps/web/src/app/tools/de-duplicate/de-duplicate.component.html b/apps/web/src/app/tools/de-duplicate/de-duplicate.component.html new file mode 100644 index 00000000000..88ebc950a95 --- /dev/null +++ b/apps/web/src/app/tools/de-duplicate/de-duplicate.component.html @@ -0,0 +1,28 @@ + + + +
+

{{ "deDuplicateVault" | i18n }}

+ + {{ "deDuplicationToolDescription" | i18n }} + + + + {{ callout.message }} + + +
+
+ +
+
+
+ diff --git a/apps/web/src/app/tools/de-duplicate/de-duplicate.component.ts b/apps/web/src/app/tools/de-duplicate/de-duplicate.component.ts new file mode 100644 index 00000000000..4f420720c88 --- /dev/null +++ b/apps/web/src/app/tools/de-duplicate/de-duplicate.component.ts @@ -0,0 +1,71 @@ +import { CommonModule } from "@angular/common"; +import { ChangeDetectorRef, Component, Inject } from "@angular/core"; +import { firstValueFrom } from "rxjs"; + +import { AccountService } from "@bitwarden/common/auth/abstractions/account.service"; +import { getUserId } from "@bitwarden/common/auth/services/account.service"; +import { I18nService } from "@bitwarden/common/platform/abstractions/i18n.service"; +import { ButtonModule, CalloutModule } from "@bitwarden/components"; +import { I18nPipe } from "@bitwarden/ui-common"; + +import { HeaderModule } from "../../layouts/header/header.module"; +import { SharedModule } from "../../shared"; +import { DeDuplicateService } from "../../vault/services/de-duplicate.service"; + +@Component({ + selector: "app-de-duplicate", + standalone: true, + imports: [CommonModule, SharedModule, HeaderModule, ButtonModule, CalloutModule, I18nPipe], + templateUrl: "./de-duplicate.component.html", +}) +export class DeDuplicateComponent { + loading = false; + callout: { type: "success" | "warning"; title?: string; message: string } | null = null; + + constructor( + @Inject(DeDuplicateService) private deDuplicateService: DeDuplicateService, + private i18nService: I18nService, + private accountService: AccountService, + private cdr: ChangeDetectorRef, + ) {} + + findDuplicates = async () => { + this.callout = null; + + // Allow progress spinner to appear on button + await new Promise((r) => setTimeout(r, 100)); + + try { + const userId = await firstValueFrom(this.accountService.activeAccount$.pipe(getUserId)); + const result = await this.deDuplicateService.findAndHandleDuplicates(userId); + + if (result.setsFound === 0) { + this.callout = { + type: "success", + title: this.i18nService.t("deDuplicationComplete"), + message: this.i18nService.t("noDuplicatesFound"), + }; + } else { + const trashed = result.trashed ?? 0; + const permanentlyDeleted = result.permanentlyDeleted ?? 0; + const parts: string[] = []; + parts.push(`${trashed} ${this.i18nService.t("itemsTrashed")}`); + parts.push(`${permanentlyDeleted} ${this.i18nService.t("itemsPermanentlyDeleted")}`); + this.callout = { + type: "success", + title: this.i18nService.t("deDuplicationComplete"), + message: parts.join(", ") + ".", + }; + } + } catch (e) { + const message = this.i18nService.t("duplicateError"); + this.callout = { + type: "warning", + title: message, + message: `${message}: ${e}`, + }; + } finally { + this.cdr.markForCheck?.(); + } + }; +} diff --git a/apps/web/src/app/tools/de-duplicate/duplicate-review-dialog.component.html b/apps/web/src/app/tools/de-duplicate/duplicate-review-dialog.component.html new file mode 100644 index 00000000000..2dd9888e4b0 --- /dev/null +++ b/apps/web/src/app/tools/de-duplicate/duplicate-review-dialog.component.html @@ -0,0 +1,127 @@ + + + {{ totalDuplicateItemCount }} {{ "duplicatesFound" | i18n }} + +
+
+ +
+ {{ "found" | i18n }} + {{ totalDuplicateItemCount }} + {{ "duplicateItemsIn" | i18n }} + {{ duplicateSetsLength }} + {{ "sets" | i18n }}. +
+ + + + + + +
+
+
{{ i + 1 }}. {{ set.displayKey }}
+
+ {{ set.ciphers.length }} + {{ set.ciphers.length === 1 ? ("item" | i18n) : ("items" | i18n) }} +
+ + + + + {{ "name" | i18n }} + + + {{ "username" | i18n }} + + + {{ "websiteUri" | i18n }} + + + {{ "folder" | i18n }} + + + {{ "organization" | i18n }} + + + + + + + + {{ + row.name + }} + + ({{ "itemInTrash" | i18n }}) + + {{ row.login?.username }} + + {{ getCipherUris(row) || ("noValue" | i18n) }} + + {{ row.folderId || ("noValue" | i18n) }} + {{ row.organizationId || ("noValue" | i18n) }} + + +
+
+ +
+
+ + + + + diff --git a/apps/web/src/app/tools/de-duplicate/duplicate-review-dialog.component.ts b/apps/web/src/app/tools/de-duplicate/duplicate-review-dialog.component.ts new file mode 100644 index 00000000000..03454a1705f --- /dev/null +++ b/apps/web/src/app/tools/de-duplicate/duplicate-review-dialog.component.ts @@ -0,0 +1,160 @@ +import { CommonModule } from "@angular/common"; +import { ChangeDetectionStrategy, Component, Inject } from "@angular/core"; +import { FormsModule } from "@angular/forms"; + +import { CipherView } from "@bitwarden/common/vault/models/view/cipher.view"; +import { + ButtonModule, + DialogModule, + DialogRef, + DIALOG_DATA, + TableDataSource, + TableModule, +} from "@bitwarden/components"; +import { I18nPipe } from "@bitwarden/ui-common"; + +export interface DuplicateReviewDialogResult { + confirmed: boolean; + deleteCipherIds: string[]; +} + +@Component({ + selector: "app-duplicate-review-dialog", + standalone: true, + imports: [CommonModule, DialogModule, ButtonModule, FormsModule, I18nPipe, TableModule], + templateUrl: "./duplicate-review-dialog.component.html", + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class DuplicateReviewDialogComponent { + selection: Record = {}; + selectedCount = 0; + dataSource: TableDataSource<{ key: string; displayKey: string; ciphers: CipherView[] }>; + + private _totalDuplicateItemCount = 0; + get totalDuplicateItemCount(): number { + return this._totalDuplicateItemCount; + } + + constructor( + private dialogRef: DialogRef, + @Inject(DIALOG_DATA) public data: { duplicateSets: { key: string; ciphers: CipherView[] }[] }, + ) { + this.dataSource = new TableDataSource<{ + key: string; + displayKey: string; + ciphers: CipherView[]; + }>(); + const sets = (data.duplicateSets ?? []).map((set) => ({ + ...set, + displayKey: this.getDuplicateSetDomain(set.key), + })); + this.dataSource.data = sets; + + for (const set of sets) { + set.ciphers.forEach((c, idx) => { + this.selection[c.id] = idx !== 0; + }); + } + + this._totalDuplicateItemCount = sets.reduce( + (sum, s) => sum + Math.max(0, s.ciphers.length - 1), + 0, + ); + this.selectedCount = Object.values(this.selection).filter(Boolean).length; + } + + /** + * Extracts the domain or grouping info from the duplicate set key. + * For keys in the format 'username+uri: username @ domain', returns domain. + * For keys in the format 'username+name: username & name', returns name. + * Otherwise returns the key itself. + */ + getDuplicateSetDomain(key: string): string { + // username+uri: username @ domain + if (key.startsWith("username+uri:")) { + const atIdx = key.lastIndexOf("@"); + if (atIdx !== -1) { + return key.substring(atIdx + 1).trim(); + } + } + // username+name: username & name + if (key.startsWith("username+name:")) { + const ampIdx = key.lastIndexOf("&"); + if (ampIdx !== -1) { + return key.substring(ampIdx + 1).trim(); + } + } + return key; + } + // Cache TableDataSource per ciphers array to avoid recreating on each change detection + private dsCache = new WeakMap>(); + getTableDataSource(ciphers: CipherView[]): TableDataSource { + if (!ciphers) { + const empty = new TableDataSource(); + empty.data = []; + return empty; + } + const cached = this.dsCache.get(ciphers); + if (cached) { + return cached; + } + const ds = new TableDataSource(); + ds.data = ciphers; + this.dsCache.set(ciphers, ds); + return ds; + } + + // Cache URI string per cipher instance + private urisCache = new WeakMap(); + getCipherUris(cipher: CipherView): string { + if (!cipher) { + return ""; + } + const cached = this.urisCache.get(cipher); + if (cached !== undefined) { + return cached; + } + const value = + cipher.login?.uris + ?.map((u) => u?.uri) + .filter((uri): uri is string => !!uri) + .join(", ") || ""; + this.urisCache.set(cipher, value); + return value; + } + + onSelectChange(id: string, selected: boolean): void { + const prev = !!this.selection[id]; + if (prev === selected) { + return; + } + this.selection[id] = selected; + this.selectedCount += selected ? 1 : -1; + } + + get anySelected(): boolean { + return this.selectedCount > 0; + } + + trackBySet(_index: number, set: { key: string }): string { + return set.key; + } + + confirm(): void { + const deleteCipherIds = Object.entries(this.selection) + .filter(([, selected]) => selected) + .map(([id]) => id); + this.dialogRef.close({ confirmed: true, deleteCipherIds }); + } + + cancel(): void { + this.dialogRef.close({ confirmed: false, deleteCipherIds: [] }); + } + + static open( + dialogService: any, + data: { duplicateSets: { key: string; ciphers: CipherView[] }[] }, + ) { + return (dialogService as any).open(DuplicateReviewDialogComponent, { data }); + } +} diff --git a/apps/web/src/app/vault/services/de-duplicate.service.spec.ts b/apps/web/src/app/vault/services/de-duplicate.service.spec.ts new file mode 100644 index 00000000000..7620cdf2d75 --- /dev/null +++ b/apps/web/src/app/vault/services/de-duplicate.service.spec.ts @@ -0,0 +1,559 @@ +import { CipherView } from "@bitwarden/common/vault/models/view/cipher.view"; + +import { DeDuplicateService } from "./de-duplicate.service"; + +// Minimal stubs (we directly exercise private logic; no need for full implementations) +const cipherServiceStub = { getAllDecrypted: jest.fn() }; +const dialogServiceStub = { open: jest.fn() }; +const cipherAuthorizationServiceStub = {}; + +type UriLike = string | { uri?: string; decryptedValue?: string; text?: string } | null | undefined; + +function buildCipher({ + id, + name, + username, + password = "p", + uris = [], +}: { + id: string; + name: string; + username: string; + password?: string; + uris?: UriLike[]; +}): CipherView { + const cv = new CipherView(); + (cv as any).id = id; + (cv as any).name = name; + (cv as any).login = { username, password, uris } as any; + return cv; +} + +describe("DeDuplicateService core duplicate detection", () => { + let service: DeDuplicateService; + const findSets = (ciphers: CipherView[]) => + (service as any).findDuplicateSets(ciphers) as { key: string; ciphers: CipherView[] }[]; + const normalize = (s: string) => (service as any).normalizeUri(s) as string; + const extract = (c: CipherView) => (service as any).extractUriStrings(c) as string[]; + + beforeEach(() => { + jest.clearAllMocks(); + service = new DeDuplicateService( + cipherServiceStub as any, + dialogServiceStub as any, + cipherAuthorizationServiceStub as any, + ); + }); + + describe("username + URI bucket", () => { + it("groups items with same username and host (ignores path/query/fragment)", () => { + const c1 = buildCipher({ + id: "1", + name: "A", + username: "user@example.com", + uris: ["https://example.com/login"], + }); + const c2 = buildCipher({ + id: "2", + name: "B", + username: "user@example.com", + uris: ["https://example.com/login?foo=1#frag"], + }); + const sets = findSets([c1, c2]); + expect(sets).toHaveLength(1); + expect(sets[0].ciphers.map((c) => c.id).sort()).toEqual(["1", "2"]); + expect(sets[0].key).toBe("username+uri: user@example.com @ example.com"); + }); + + it("groups when only path differs", () => { + const c1 = buildCipher({ + id: "1", + name: "A", + username: "u", + uris: ["https://example.com/a"], + }); + const c2 = buildCipher({ + id: "2", + name: "B", + username: "u", + uris: ["https://example.com/b"], + }); + const sets = findSets([c1, c2]); + expect(sets).toHaveLength(1); + expect(sets[0].key).toBe("username+uri: u @ example.com"); + }); + + it("groups when only query differs", () => { + const c1 = buildCipher({ + id: "1", + name: "A", + username: "u", + uris: ["https://example.com/path?x=1"], + }); + const c2 = buildCipher({ + id: "2", + name: "B", + username: "u", + uris: ["https://example.com/path?x=2"], + }); + const sets = findSets([c1, c2]); + expect(sets).toHaveLength(1); + expect(sets[0].key).toBe("username+uri: u @ example.com"); + }); + + it("groups when only fragment differs", () => { + const c1 = buildCipher({ + id: "1", + name: "A", + username: "u", + uris: ["https://example.com/path#one"], + }); + const c2 = buildCipher({ + id: "2", + name: "B", + username: "u", + uris: ["https://example.com/path#two"], + }); + const sets = findSets([c1, c2]); + expect(sets).toHaveLength(1); + expect(sets[0].key).toBe("username+uri: u @ example.com"); + }); + + it("groups when only scheme differs (http vs https)", () => { + const c1 = buildCipher({ + id: "1", + name: "A", + username: "u", + uris: ["http://example.com/login"], + }); + const c2 = buildCipher({ + id: "2", + name: "B", + username: "u", + uris: ["https://example.com/login"], + }); + const sets = findSets([c1, c2]); + expect(sets).toHaveLength(1); + expect(sets[0].key).toBe("username+uri: u @ example.com"); + }); + + it("groups when host case and trailing slash differ", () => { + const c1 = buildCipher({ id: "1", name: "A", username: "u", uris: ["HTTPS://EXAMPLE.COM/"] }); + const c2 = buildCipher({ id: "2", name: "B", username: "u", uris: ["https://example.com"] }); + const sets = findSets([c1, c2]); + expect(sets).toHaveLength(1); + expect(sets[0].key).toBe("username+uri: u @ example.com"); + }); + + it("does NOT group when hosts differ even with matching username", () => { + const c1 = buildCipher({ + id: "1", + name: "X", + username: "u", + uris: ["https://a.example.com"], + }); + const c2 = buildCipher({ + id: "2", + name: "Y", + username: "u", + uris: ["https://b.example.com"], + }); + expect(findSets([c1, c2])).toHaveLength(0); + }); + + it("does NOT group when usernames differ even if host matches", () => { + const c1 = buildCipher({ + id: "1", + name: "A", + username: "alice", + uris: ["https://service.test"], + }); + const c2 = buildCipher({ + id: "2", + name: "B", + username: "bob", + uris: ["http://service.test/"], + }); + expect(findSets([c1, c2])).toHaveLength(0); + }); + + it("multiple URIs per cipher can map to multiple hosts, but identical membership collapses to one set", () => { + const c1 = buildCipher({ + id: "1", + name: "A", + username: "u", + uris: ["https://a.test", "https://b.test/"], + }); + const c2 = buildCipher({ + id: "2", + name: "B", + username: "u", + uris: ["http://A.test/", "http://b.test"], + }); + const sets = findSets([c1, c2]); + expect(sets).toHaveLength(1); + const key = sets[0].key; + expect(["username+uri: u @ a.test", "username+uri: u @ b.test"]).toContain(key); + expect(sets[0].ciphers.map((c) => c.id).sort()).toEqual(["1", "2"]); + }); + + it("creates a single set when 3+ ciphers share same username+host", () => { + const cs = [1, 2, 3].map((i) => + buildCipher({ + id: String(i), + name: "N" + i, + username: "u", + uris: ["example.org/path" + i], + }), + ); + const sets = findSets(cs); + expect(sets).toHaveLength(1); + expect(sets[0].ciphers).toHaveLength(3); + expect(new Set(sets[0].ciphers.map((c) => c.id))).toEqual(new Set(["1", "2", "3"])); + expect(sets[0].key).toBe("username+uri: u @ example.org"); + }); + + it("ignores ciphers without username or without valid URIs", () => { + const good1 = buildCipher({ id: "1", name: "A", username: "u", uris: ["https://good.test"] }); + const badNoUsername = buildCipher({ id: "2", name: "B", username: "", uris: ["good.test"] }); + const badWhitespaceUsername = buildCipher({ + id: "3", + name: "C", + username: " ", + uris: ["good.test"], + }); + const badNoUris = buildCipher({ id: "4", name: "D", username: "u", uris: [] }); + const sets = findSets([good1, badNoUsername, badWhitespaceUsername, badNoUris]); + expect(sets).toHaveLength(0); // only one valid item -> no duplicate set + }); + + it("does not create a duplicate set when a single cipher has multiple URIs that normalize to the same host", () => { + const c1 = buildCipher({ + id: "1", + name: "!_TEST", + username: "tester", + uris: [ + "forum.test.domain.org", + "forum.test.domain.org/login", + "HTTP://FORUM.TEST.DOMAIN.ORG", + ], + }); + // Only one cipher overall; previously this could push the same cipher twice into the same bucket. + const sets = findSets([c1]); + expect(sets).toHaveLength(0); + }); + + it("with multiple items, a cipher that lists the same host multiple times appears only once in that host's grouping", () => { + const c1 = buildCipher({ + id: "1", + name: "!_TEST", + username: "tester", + uris: ["test.domain.org", "forum.test.domain.org", "forum.test.domain.org/login"], + }); + const c2 = buildCipher({ + id: "2", + name: "2_TEST", + username: "tester", + uris: ["test.domain.org"], + }); + const c3 = buildCipher({ + id: "3", + name: "3_TEST", + username: "tester", + uris: ["forum.test.domain.org"], + }); + + const sets = findSets([c1, c2, c3]); + const setsByKey = new Map(sets.map((s) => [s.key, s])); + + const forumKey = "username+uri: tester @ forum.test.domain.org"; + const testKey = "username+uri: tester @ test.domain.org"; + + expect(setsByKey.has(forumKey)).toBe(true); + expect(setsByKey.has(testKey)).toBe(true); + + const forumSet = setsByKey.get(forumKey)!; + const testSet = setsByKey.get(testKey)!; + + // Each set should contain each cipher at most once + expect(forumSet.ciphers.map((c) => c.id).sort()).toEqual(["1", "3"]); + expect(testSet.ciphers.map((c) => c.id).sort()).toEqual(["1", "2"]); + }); + }); + + describe("username + name bucket", () => { + it("groups on username+name when at least two ciphers match and username+uri doesn't apply", () => { + const c1 = buildCipher({ + id: "1", + name: "Shared Name", + username: "u", + uris: ["https://one.example"], + }); + const c2 = buildCipher({ + id: "2", + name: "Shared Name", + username: "u", + uris: ["https://two.example"], + }); + // Hosts differ so no URI duplicate; should produce exactly one name-based set + const sets = findSets([c1, c2]); + expect(sets).toHaveLength(1); + expect(sets[0].key).toBe("username+name: u & Shared Name"); + expect(new Set(sets[0].ciphers.map((c) => c.id))).toEqual(new Set(["1", "2"])); + }); + + it("groups when names differ only by case (case-insensitive)", () => { + const c1 = buildCipher({ id: "1", name: "Login", username: "u", uris: ["a.example"] }); + const c2 = buildCipher({ id: "2", name: "login", username: "u", uris: ["b.example"] }); + const sets = findSets([c1, c2]); + const nameSet = sets.find((s) => s.key.startsWith("username+name: u &")); + expect(nameSet).toBeDefined(); + expect(new Set(nameSet!.ciphers.map((c) => c.id))).toEqual(new Set(["1", "2"])); + }); + + it("groups when names match after trimming outer whitespace", () => { + const c1 = buildCipher({ id: "1", name: " Space ", username: "u", uris: ["one.example"] }); + const c2 = buildCipher({ id: "2", name: "Space", username: "u", uris: ["two.example"] }); + const sets = findSets([c1, c2]); + const match = sets.find((s) => s.key === "username+name: u & Space"); + expect(match).toBeDefined(); + expect(new Set(match!.ciphers.map((c) => c.id))).toEqual(new Set(["1", "2"])); + }); + + it("groups when internal whitespace differs (whitespace-insensitive)", () => { + const c1 = buildCipher({ id: "1", name: "My Site", username: "u", uris: ["x.example"] }); + const c2 = buildCipher({ id: "2", name: "My\tSite", username: "u", uris: ["y.example"] }); + const c3 = buildCipher({ id: "3", name: "My Site", username: "u", uris: ["z.example"] }); + const sets = findSets([c1, c2, c3]); + const nameSet = sets.find((s) => s.key.startsWith("username+name: u &")); + expect(nameSet).toBeDefined(); + expect(new Set(nameSet!.ciphers.map((c) => c.id))).toEqual(new Set(["1", "2", "3"])); + }); + + it("does NOT group when usernames differ only by case (username case sensitive)", () => { + const c1 = buildCipher({ id: "1", name: "Exact", username: "User", uris: ["site1.example"] }); + const c2 = buildCipher({ id: "2", name: "Exact", username: "user", uris: ["site2.example"] }); + const sets = findSets([c1, c2]); + expect(sets.find((s) => s.key.startsWith("username+name:"))).toBeUndefined(); + }); + + it("groups when usernames match after trimming outer whitespace", () => { + const c1 = buildCipher({ id: "1", name: "Label", username: " user ", uris: ["h1.example"] }); + const c2 = buildCipher({ id: "2", name: "Label", username: "user", uris: ["h2.example"] }); + const sets = findSets([c1, c2]); + const match = sets.find((s) => s.key === "username+name: user & Label"); + expect(match).toBeDefined(); + expect(new Set(match!.ciphers.map((c) => c.id))).toEqual(new Set(["1", "2"])); + }); + + it("creates a single name bucket for 3+ matching ciphers", () => { + const cs = [1, 2, 3].map((i) => + buildCipher({ + id: String(i), + name: "Cluster", + username: "u", + uris: ["host" + i + ".example"], + }), + ); + const sets = findSets(cs); + const nameSet = sets.filter((s) => s.key === "username+name: u & Cluster"); + expect(nameSet).toHaveLength(1); + expect(nameSet[0].ciphers).toHaveLength(3); + }); + + it("includes ciphers lacking any URIs in name bucket grouping", () => { + const c1 = buildCipher({ id: "1", name: "Shared", username: "u", uris: [] }); + const c2 = buildCipher({ id: "2", name: "Shared", username: "u", uris: ["alpha.example"] }); + const sets = findSets([c1, c2]); + const match = sets.find((s) => s.key === "username+name: u & Shared"); + expect(match).toBeDefined(); + expect(new Set(match!.ciphers.map((c) => c.id))).toEqual(new Set(["1", "2"])); + }); + + it("can produce both URI and name duplicate sets (distinct keys, overlapping ciphers)", () => { + const c1 = buildCipher({ id: "1", name: "Same", username: "u", uris: ["a.example"] }); + const c2 = buildCipher({ id: "2", name: "Same", username: "u", uris: ["A.EXAMPLE/path"] }); + const c3 = buildCipher({ id: "3", name: "Same", username: "u", uris: ["other.example"] }); + const sets = findSets([c1, c2, c3]); + const keys = sets.map((s) => s.key).sort(); + expect(keys).toEqual(["username+name: u & Same", "username+uri: u @ a.example"]); + const uriSet = sets.find((s) => s.key === "username+uri: u @ a.example")!; + const nameSet = sets.find((s) => s.key === "username+name: u & Same")!; + expect(new Set(uriSet.ciphers.map((c) => c.id))).toEqual(new Set(["1", "2"])); + expect(new Set(nameSet.ciphers.map((c) => c.id))).toEqual(new Set(["1", "2", "3"])); + }); + + it("when URI and name sets have identical cipher IDs, only the URI set is kept", () => { + const c1 = buildCipher({ id: "1", name: "Same", username: "u", uris: ["a.example"] }); + const c2 = buildCipher({ id: "2", name: "Same", username: "u", uris: ["A.EXAMPLE"] }); + const sets = findSets([c1, c2]); + // both groupings would include [1,2], but the implementation prefers the URI set + expect(sets).toHaveLength(1); + expect(sets[0].key).toBe("username+uri: u @ a.example"); + expect(new Set(sets[0].ciphers.map((c) => c.id))).toEqual(new Set(["1", "2"])); + }); + + it("does not create name bucket if only one cipher present", () => { + const c1 = buildCipher({ id: "1", name: "Solo", username: "u", uris: ["solo.example"] }); + expect(findSets([c1])).toHaveLength(0); + }); + + it("requires non-empty trimmed username and name", () => { + const good = buildCipher({ id: "1", name: "Name", username: "u", uris: ["x.test"] }); + const noName = buildCipher({ id: "2", name: "", username: "u", uris: ["y.test"] }); + const blankName = buildCipher({ id: "3", name: " ", username: "u", uris: ["z.test"] }); + const noUsername = buildCipher({ id: "4", name: "Name", username: "", uris: ["z.test"] }); + expect(findSets([good, noName, blankName, noUsername])).toHaveLength(0); + }); + }); + + describe("URI extraction", () => { + it("extracts from string and object variants (uri, decryptedValue, text) and ignores empty/missing", () => { + const c = buildCipher({ + id: "1", + name: "X", + username: "u", + uris: [ + "https://one.test/path", + { uri: "two.test" }, + { decryptedValue: "http://three.test/a" }, + { text: "FOUR.test" }, + { uri: "" }, + null, + undefined, + {}, + ], + }); + const values = extract(c).sort(); + expect(values).toEqual( + ["FOUR.test", "http://three.test/a", "https://one.test/path", "two.test"].sort(), + ); + }); + + it("returns empty array when login.uris is not an array", () => { + const c = buildCipher({ id: "1", name: "X", username: "u", uris: [] }); + // Force a non-array value + (c as any).login.uris = "not-an-array" as any; + expect(extract(c)).toEqual([]); + }); + }); + + describe("normalizeUri", () => { + it("adds https scheme when missing", () => { + expect(normalize("Example.com/login?x=1")).toBe("example.com"); + }); + + it("lowercases host and strips path/query/fragment", () => { + expect(normalize("HTTP://EXAMPLE.COM/Path/To?x=1#hash")).toBe("example.com"); + }); + + it("strips default/any port", () => { + expect(normalize("https://example.com:443/foo")).toBe("example.com"); + expect(normalize("https://example.com:8443/foo")).toBe("example.com"); + }); + + it("handles IPv4", () => { + expect(normalize("http://192.168.0.1/login")).toBe("192.168.0.1"); + }); + + it("handles IPv6 with brackets and port", () => { + expect(normalize("http://[2001:db8::1]:8080/path")).toBe("2001:db8::1"); + }); + + it("removes trailing dot", () => { + expect(normalize("https://example.com./x")).toBe("example.com"); + }); + + it("removes userinfo", () => { + expect(normalize("https://user:pass@example.com/path")).toBe("example.com"); + }); + + it("returns empty string for blank input", () => { + expect(normalize("")).toBe(""); + expect(normalize(" ")).toBe(""); + }); + + it("supports internationalized domains via punycode", () => { + // The URL parser returns punycoded hostname for IDN. + const host = normalize("https://münich.example/secure"); + expect(host).toMatch(/^xn--mnich-kva\.example$/); // Punycode of münich.example + }); + + describe("fallback regex path (forced URL parse failure)", () => { + let originalURL: any; + beforeEach(() => { + originalURL = (global as any).URL; + (global as any).URL = class FailingURL { + constructor(_s: string) { + throw new Error("forced parse failure"); + } + } as any; + }); + afterEach(() => { + (global as any).URL = originalURL; + }); + + it("extracts host + lowercases + strips path/query/fragment", () => { + expect(normalize("HTTPS://Example.COM/Some/Path?x=1#frag")).toBe("example.com"); + }); + + it("strips userinfo and port", () => { + expect(normalize("custom+scheme://user:pass@Sub.Domain.Example.COM:8080/resource")).toBe( + "sub.domain.example.com", + ); + }); + + it("handles IPv6 with brackets and port", () => { + expect(normalize("ftp://[2001:db8::2]:8042/over/there")).toBe("2001:db8::2"); + }); + + it("removes trailing dot", () => { + expect(normalize("https://example.com./path")).toBe("example.com"); + }); + + it("returns empty string when authority missing", () => { + expect(normalize("https://")).toBe(""); + }); + }); + }); + + describe("name-only bucket (no username)", () => { + it("groups items without username by canonicalized name (case/whitespace-insensitive)", () => { + const c1 = buildCipher({ id: "1", name: "My App", username: "", uris: [] }); + const c2 = buildCipher({ id: "2", name: " my\tapp ", username: "", uris: [] }); + const sets = findSets([c1, c2]); + expect(sets).toHaveLength(1); + expect(sets[0].key).toBe("username+name: & My App"); + expect(new Set(sets[0].ciphers.map((c) => c.id))).toEqual(new Set(["1", "2"])); + }); + + it("does not mix username-present with username-absent, even if names match", () => { + const noUser = buildCipher({ id: "1", name: "Same", username: "", uris: [] }); + const u1 = buildCipher({ id: "2", name: "Same", username: "u", uris: [] }); + const u2 = buildCipher({ id: "3", name: "Same", username: "u", uris: [] }); + const sets = findSets([noUser, u1, u2]); + // Should only produce the username+name set for user "u" + const keys = sets.map((s) => s.key).sort(); + expect(keys).toEqual(["username+name: u & Same"]); + const unameSet = sets[0]; + expect(new Set(unameSet.ciphers.map((c) => c.id))).toEqual(new Set(["2", "3"])); + }); + + it("creates a single name-only bucket for 3+ matching ciphers", () => { + const cs = [1, 2, 3].map((i) => + buildCipher({ id: String(i), name: "Cluster", username: "", uris: [] }), + ); + const sets = findSets(cs); + expect(sets).toHaveLength(1); + expect(sets[0].key).toBe("username+name: & Cluster"); + expect(new Set(sets[0].ciphers.map((c) => c.id))).toEqual(new Set(["1", "2", "3"])); + }); + + it("does not create a name-only set for a single item or blank name", () => { + const solo = buildCipher({ id: "1", name: "Solo", username: "", uris: [] }); + const blank = buildCipher({ id: "2", name: " ", username: "", uris: [] }); + expect(findSets([solo])).toHaveLength(0); + expect(findSets([blank])).toHaveLength(0); + }); + }); +}); diff --git a/apps/web/src/app/vault/services/de-duplicate.service.ts b/apps/web/src/app/vault/services/de-duplicate.service.ts new file mode 100644 index 00000000000..673836f6910 --- /dev/null +++ b/apps/web/src/app/vault/services/de-duplicate.service.ts @@ -0,0 +1,363 @@ +import { Injectable } from "@angular/core"; +import { firstValueFrom } from "rxjs"; + +import { UserId } from "@bitwarden/common/types/guid"; +import { CipherService } from "@bitwarden/common/vault/abstractions/cipher.service"; +import { CipherView } from "@bitwarden/common/vault/models/view/cipher.view"; +import { CipherAuthorizationService } from "@bitwarden/common/vault/services/cipher-authorization.service"; +import { DialogService } from "@bitwarden/components"; + +import { + DuplicateReviewDialogComponent, + DuplicateReviewDialogResult, +} from "../../tools/de-duplicate/duplicate-review-dialog.component"; +// Success dialog replaced by callout shown in the de-duplicate component + +export interface DuplicateOperationResult { + setsFound: number; + trashed: number; + permanentlyDeleted: number; +} + +interface DuplicateSet { + key: string; + ciphers: CipherView[]; +} + +@Injectable({ + providedIn: "root", +}) +export class DeDuplicateService { + constructor( + private cipherService: CipherService, + private dialogService: DialogService, + private cipherAuthorizationService: CipherAuthorizationService, + ) {} + + /** + * Main entry point to find and handle duplicate ciphers for a given user. + * @param userId The ID of the current user. + * @returns A promise that resolves to the number of duplicate sets found. + */ + async findAndHandleDuplicates(userId: UserId): Promise { + const allCiphers = await this.cipherService.getAllDecrypted(userId); + const duplicateSets = this.findDuplicateSets(allCiphers); + + if (duplicateSets.length > 0) { + const { trashed, permanentlyDeleted } = await this.handleDuplicates(duplicateSets, userId); + return { setsFound: duplicateSets.length, trashed, permanentlyDeleted }; + } + return { setsFound: 0, trashed: 0, permanentlyDeleted: 0 }; + } + + /** + * Finds groups of ciphers (clusters) that are considered duplicates. + * A "group" or cluster is defined by ciphers sharing login.username and a normalized login.uri, + * OR a matching login.username and normalized cipher.name, + * OR when no username is present, on matching normalized cipher.name only. + * @param ciphers A list of all the user's ciphers. + * @returns An array of DuplicateSet objects, each representing a group of duplicates. + */ + private findDuplicateSets(ciphers: CipherView[]): DuplicateSet[] { + const uriBuckets = new Map(); + const nameBuckets = new Map(); + const nameOnlyBuckets = new Map(); // used in edge cases when no useername is present for a login + + // DuplicateSet will be created to hold duplicate login ciphers as soon as two matching ci\hers appear in a bucket + const duplicateSets: DuplicateSet[] = []; + + // Used to prevent redundant groupings for a given display key ['username+uri', 'username+name'] + // Note that matchings based solely on name (no username for login) will share the 'username+name' display key + const setByDisplayKey = new Map(); + + /** + * When a bucket first qualifies as a duplicate (size === 2), create a single DuplicateSet, + * register it by displayKey to prevent redundant groupings, and reuse the bucket array + * so later additions are reflected. + * + * @param bucket Accumulated ciphers for this grouping key. + * @param displayKey Human-friendly label identifying the group. + */ + const ensureSetForBucket = (bucket: CipherView[], displayKey: string): void => { + if (bucket.length === 2 && !setByDisplayKey.has(displayKey)) { + const ds: DuplicateSet = { key: displayKey, ciphers: bucket }; + setByDisplayKey.set(displayKey, ds); + duplicateSets.push(ds); + } + }; + + for (const cipher of ciphers) { + const username = cipher.login?.username?.trim() || ""; + + // Match URIs when username is present + // Almost all dudplicates can be identified by matching username and URI - other cases handled in next block + if (username) { + const uris = this.extractUriStrings(cipher); + if (uris.length > 0) { + // Collect unique normalized hosts to avoid adding the same cipher twice to the same bucket + const hosts = new Set(); + for (const uri of uris) { + const normHost = this.normalizeUri(uri); + if (normHost) { + hosts.add(normHost); + } + } + for (const normHost of hosts) { + const key = `${username}||${normHost}`; + let bucket = uriBuckets.get(key); + if (!bucket) { + bucket = []; + uriBuckets.set(key, bucket); + } + bucket.push(cipher); + const displayKey = `username+uri: ${username} @ ${normHost}`; + ensureSetForBucket(bucket, displayKey); // Create/extend duplicate set when bucket reaches size 2 + } + } + } + + // Match on names, with or without usernames + const rawName = cipher.name?.trim(); + if (rawName) { + const canonical = this.canonicalizeName(rawName); + if (canonical) { + if (username) { + const key = `${username}||${canonical}`; + let bucket = nameBuckets.get(key); + if (!bucket) { + bucket = []; + nameBuckets.set(key, bucket); + } + bucket.push(cipher); + const displayName = bucket[0].name?.trim() || ""; + const displayKey = `username+name: ${username} & ${displayName}`; + ensureSetForBucket(bucket, displayKey); // Create/extend duplicate set when bucket reaches size 2 + } else { + // match on cipher.name only when username is absent + // to prevent false positive duplicates in a situation where a user has multiple accounts on the same site - among others + let bucket = nameOnlyBuckets.get(canonical); + if (!bucket) { + bucket = []; + nameOnlyBuckets.set(canonical, bucket); + } + bucket.push(cipher); + const displayName = bucket[0].name?.trim() || ""; + // Reuse existing display format so UI logic extracts the name without introducing new labels + const displayKey = `username+name: & ${displayName}`; + ensureSetForBucket(bucket, displayKey); // Create/extend duplicate set when bucket reaches size 2 + } + } + } + } + + // Collapse groups that contain the exact same cipher IDs + // Prefer the stronger username+uri grouping over username+name + const weightedDuplicateSets = new Map(); // used to prioritize username+uri ses + const groupingPriority = (key: string): number => (key.startsWith("username+uri:") ? 2 : 1); + + for (const set of duplicateSets) { + const signature = set.ciphers + .map((c) => c.id) + .sort() + .join("|"); // string representing ciphr IDs in a reproducible way + const existing = weightedDuplicateSets.get(signature); + if (!existing || groupingPriority(set.key) > groupingPriority(existing.key)) { + weightedDuplicateSets.set(signature, set); + } + } + + return Array.from(weightedDuplicateSets.values()); + } + + /** + * Produce a canonical form of a name for duplicate comparison: + * - Remove ALL whitespace characters (internal & external). + * - Lowercase the result. + * - Return empty string if nothing remains. + */ + private canonicalizeName(name: string): string { + return name.replace(/\s+/g, "").toLowerCase(); + } + + /** + * Returns all URI strings from a login. + * Handles both string and object entries in the `uris` array. + * Ignores empty or invalid entries. + * + * @param cipher The cipher to extract URIs from. + * @returns Array of URI strings. + */ + private extractUriStrings(cipher: CipherView): string[] { + const uris = (cipher.login as any)?.uris; + if (!uris || !Array.isArray(uris)) { + return []; + } + const out: string[] = []; + for (const entry of uris) { + if (!entry) { + continue; + } + if (typeof entry === "string") { + out.push(entry); + } else if (typeof entry === "object") { + const value = (entry.uri ?? entry.decryptedValue ?? entry.text ?? "") as string; + if (typeof value === "string" && value.trim().length > 0) { + out.push(value); + } + } + } + return out; + } + + /** + * Extracts the host portion (subdomains.domain.tld OR IPv4 OR IPv6) from an input string. + * Behavior: + * - Prepends "https://" if the string lacks a scheme so standard parsing works. + * - Uses the node's URL parser when available (new URL). That yields punycoded ASCII for IDNs. + * - Falls back to a lightweight regex authority parse if URL parsing fails or isn't available. + * - Strips userinfo, port, enclosing IPv6 brackets, and a trailing dot; lowercases result. + * - Returns "" if a host can't be derived. + * @param raw Input possibly containing a host. + * @returns Host string or empty string. + */ + private normalizeUri(raw: string): string { + if (!raw) { + return ""; + } + let input = raw.trim(); + if (!input) { + return ""; + } + if (!/^[a-z][a-z0-9+.-]*:\/\//i.test(input)) { + input = "https://" + input; + } + // Attempt extraction using node's URL lib + try { + const url = new URL(input); + let host = url.hostname || ""; // hostname excludes port already + if (!host) { + return ""; + } + // Strip IPv6 brackets + if (host.startsWith("[") && host.endsWith("]")) { + host = host.slice(1, -1); + } + host = host.replace(/\.$/, "").toLowerCase(); + return host; + } catch { + // Fallback: manual authority extraction + const authorityMatch = input.match(/^[a-z][a-z0-9+.-]*:\/\/([^/?#]+)/i); + if (!authorityMatch) { + return ""; + } + let authority = authorityMatch[1]; + // Strip userinfo if present (user:pass@host) + const atIndex = authority.lastIndexOf("@"); + if (atIndex !== -1) { + authority = authority.slice(atIndex + 1); + } + // IPv6 brackets + if (authority.startsWith("[") && authority.includes("]")) { + authority = authority.slice(1, authority.indexOf("]")); + } else { + // Port (last colon, numeric part) + const c = authority.lastIndexOf(":"); + if (c !== -1 && /^[0-9]+$/.test(authority.slice(c + 1))) { + authority = authority.slice(0, c); + } + } + authority = authority.replace(/\.$/, "").toLowerCase(); + return authority; + } + } + + /** + * Handles the user interaction and server-side deletion of identified duplicates. + * This method prompts the user, checks permissions, and performs batch deletions. + * @param duplicateSets The groups of duplicate ciphers found earlier. + * @param userId The ID of the current user. + */ + private async handleDuplicates( + duplicateSets: DuplicateSet[], + userId: UserId, + ): Promise<{ trashed: number; permanentlyDeleted: number }> { + // 1. Open the dialog to let the user review and select duplicates to delete. + const dialogRef = DuplicateReviewDialogComponent.open(this.dialogService, { + duplicateSets, + }); + + const result: DuplicateReviewDialogResult | undefined = await firstValueFrom(dialogRef.closed); + if (!result?.confirmed || result.deleteCipherIds.length === 0) { + return { trashed: 0, permanentlyDeleted: 0 }; + } + + // Avoid double-processing the same cipher when it appears in multiple duplicate sets + const uniqueDeleteIds = Array.from(new Set(result.deleteCipherIds)); + + // 2. Create a quick lookup map for the ciphers to be deleted. + const cipherIndex = new Map(); + for (const set of duplicateSets) { + for (const c of set.ciphers) { + cipherIndex.set(c.id, c); + } + } + + // 3. Filter the user's selected deletions based on their permissions. + // TODO: Determine why a user may not have permission to delete a cipher and explore ways of notifying them of this situation + const permissionChecks = uniqueDeleteIds.map(async (id) => { + const cipher = cipherIndex.get(id); + if (!cipher) { + return null; + } + const canDelete = await firstValueFrom( + this.cipherAuthorizationService.canDeleteCipher$(cipher), + ); + return canDelete ? cipher : null; + }); + const permitted = (await Promise.all(permissionChecks)).filter( + (c): c is CipherView => c != null, + ); + if (permitted.length === 0) { + return { trashed: 0, permanentlyDeleted: 0 }; + } + + // 4. Separate permitted deletions into soft-delete (to trash) and permanent-delete. + const toSoftDelete: string[] = []; + const toPermanentlyDelete: string[] = []; + for (const cipher of permitted) { + if (cipher.isDeleted) { + toPermanentlyDelete.push(cipher.id); + } else { + toSoftDelete.push(cipher.id); + } + } + + // 5. Perform the server-backed deletions in batches to avoid payload limits of > 500 ciphers + const BATCH_SIZE = 500; + const processBatches = async ( + ids: string[], + action: (batch: string[]) => Promise, + ): Promise => { + for (let i = 0; i < ids.length; i += BATCH_SIZE) { + const slice = ids.slice(i, i + BATCH_SIZE); + if (slice.length) { + await action(slice); + } + } + }; + + if (toSoftDelete.length > 0) { + await processBatches(toSoftDelete, (batch) => + this.cipherService.softDeleteManyWithServer(batch, userId), + ); + } + if (toPermanentlyDelete.length > 0) { + await processBatches(toPermanentlyDelete, (batch) => + this.cipherService.deleteManyWithServer(batch, userId), + ); + } + + // 6. Return summary to display in a callout by the caller. + return { trashed: toSoftDelete.length, permanentlyDeleted: toPermanentlyDelete.length }; + } +} diff --git a/apps/web/src/locales/en/messages.json b/apps/web/src/locales/en/messages.json index fb85d4e3dd9..74410d4d141 100644 --- a/apps/web/src/locales/en/messages.json +++ b/apps/web/src/locales/en/messages.json @@ -11016,5 +11016,65 @@ }, "showLess": { "message": "Show less" + }, + "duplicatesFound": { + "message": "duplicates found.", + "description": "A string shown after a numeric value, indicating the numeric values corresponds to the number of duplicate login items present in their vault." + }, + "found": { + "message": "Found", + "description": "A single word translation used to convey the meaning of the English words found, encountered, or discovered. Capitalized as it is used in the beginning of a sentence." + }, + "duplicateItemsIn": { + "message": "duplicate logins in", + "description": "A string used to convey context regarding the results of querying a vault for duplicate login items." + }, + "sets": { + "message": "sets", + "description": "A single word translation used to convey the meaning of the English word set, group, or cluster (plural)." + }, + "itemInTrash": { + "message": "in trash", + "description": "A string used to indicate that a login item displayed in a duplicate review dialog has been moved to the trash, lower case as it is used to add context outside of a complete sentence." + }, + "noDuplicatesFound": { + "message": "No duplicates found.", + "description": "A message shown to users indicating that no duplicate logins were discovered in their vault." + }, + "duplicateError": { + "message": "An error occurred while finding duplicates", + "description": "Displayed using ToastService to indicate an error occurred while finding duplicates." + }, + "deDuplicateVault": { + "message": "De-duplicate vault", + "description": "Displayed in a header element and bit-callout to convey the purpose of the de-duplication tool." + }, + "deDuplicationToolDescription": { + "message": "Login items that have matching usernames and websites will be made available for review and optional deletion. Deleting duplicate items not in the trash will move these items to the trash, and deleting items already in the trash will permanently remove them from your vault.", + "description": "A message explaining what the de-duplication tool will do." + }, + "findDuplicates": { + "message": "Find duplicates", + "description": "Displayed inside of a button to indicate that clicking the button will initiate a search for duplicate login items." + }, + "itemsTrashed": { + "message": "items moved to trash", + "description": "A string indicating that the preceding numeric value corresponds to a quantity of items moved to the vault trash." + }, + "itemsPermanentlyDeleted": { + "message": "items permanently deleted", + "description": "A string indicating that the preceding numeric value corresponds to the quantity of items permanently deleted." + }, + "deDuplicationComplete": { + "message": "De-duplication complete", + "description": "A message indicating that the de-duplication tool completed any requested deletions." + }, + "deDuplicationInProgress": { + "message": "Searching for duplicates…", + "description": "A message informing users that their vault is being searched for duplicate logins." + }, + "noValue": { + "message": "-", + "description": "Indicates that no value is present for a given item. First implemented when displaying a list of ciphers that may not belong to folders or organizations." } }