1
0
mirror of https://github.com/bitwarden/browser synced 2025-12-15 15:53:27 +00:00

[PM-328] Move exporter to tools (#5070)

* Create and register new libs/exporter

Create package.json
Create tsconfig
Create jest.config
Extend shared and root tsconfig and jest.configs
Register with eslint

* Migrate exportService to libs/exporter

Move exportService (abstraction and impl) into libs/exporter
Refactored exportService to be split into vault-export and event-export
Created barrel-files for both exports
Moved export.service.spec.ts into vault-export
Created an export-helper, which helps build the filename (extract method refactor from ExportService)

* Move components in libs/angular into tools-subfolder

Moved components
Updated imports in jslib-services.module and jslib.module

* Register libs/exporter with browser and fix imports

Move export.component into tools-subfolder

* Register libs/exporter with cli and fix imports

Move export.command into tools-subfolder

* Register libs/exporter with desktop and fix imports

Move export.component into tools-subfolder

* Move export models to libs/exporter

* Update web imports

* Update package-lock.json

* Move export models back as it would create circular dependency

Reponse models in common rely on export models which are in libs/exporter, which relies on common

* Fix up web for event-export

* Update CODEOWNERS

* Add export-models to team-tools-dev

* Simplify domain import

* Moving EventExport into web
This commit is contained in:
Daniel James Smith
2023-04-19 11:30:46 +02:00
committed by GitHub
parent 830af7b06d
commit 192bb5a7b3
53 changed files with 266 additions and 133 deletions

View File

@@ -0,0 +1,24 @@
export class ExportHelper {
static getFileName(prefix: string = null, extension = "csv"): string {
const now = new Date();
const dateString =
now.getFullYear() +
"" +
this.padNumber(now.getMonth() + 1, 2) +
"" +
this.padNumber(now.getDate(), 2) +
this.padNumber(now.getHours(), 2) +
"" +
this.padNumber(now.getMinutes(), 2) +
this.padNumber(now.getSeconds(), 2);
return "bitwarden" + (prefix ? "_" + prefix : "") + "_export_" + dateString + "." + extension;
}
private static padNumber(num: number, width: number, padCharacter = "0"): string {
const numString = num.toString();
return numString.length >= width
? numString
: new Array(width - numString.length + 1).join(padCharacter) + numString;
}
}

View File

@@ -0,0 +1,11 @@
export interface BitwardenPasswordProtectedFileFormat {
encrypted: boolean;
passwordProtected: boolean;
salt: string;
kdfIterations: number;
kdfMemory?: number;
kdfParallelism?: number;
kdfType: number;
encKeyValidation_DO_NOT_EDIT: string;
data: string;
}

View File

@@ -0,0 +1,2 @@
export * from "./services/vault-export.service.abstraction";
export * from "./services/vault-export.service";

View File

@@ -0,0 +1,9 @@
export const EXPORT_FORMATS = ["csv", "json", "encrypted_json"] as const;
export type ExportFormat = (typeof EXPORT_FORMATS)[number];
export abstract class VaultExportServiceAbstraction {
getExport: (format?: ExportFormat, organizationId?: string) => Promise<string>;
getPasswordProtectedExport: (password: string, organizationId?: string) => Promise<string>;
getOrganizationExport: (organizationId: string, format?: ExportFormat) => Promise<string>;
getFileName: (prefix?: string, extension?: string) => string;
}

View File

@@ -0,0 +1,293 @@
// eslint-disable-next-line no-restricted-imports
import { Arg, Substitute, SubstituteOf } from "@fluffy-spoon/substitute";
import { ApiService } from "@bitwarden/common/abstractions/api.service";
import { CryptoService } from "@bitwarden/common/abstractions/crypto.service";
import { CryptoFunctionService } from "@bitwarden/common/abstractions/cryptoFunction.service";
import { KdfConfig } from "@bitwarden/common/auth/models/domain/kdf-config";
import { KdfType, DEFAULT_PBKDF2_ITERATIONS } from "@bitwarden/common/enums";
import { Utils } from "@bitwarden/common/misc/utils";
import { EncString } from "@bitwarden/common/models/domain/enc-string";
import { CipherWithIdExport } from "@bitwarden/common/models/export/cipher-with-ids.export";
import { StateService } from "@bitwarden/common/services/state.service";
import { CipherService } from "@bitwarden/common/vault/abstractions/cipher.service";
import { FolderService } from "@bitwarden/common/vault/abstractions/folder/folder.service.abstraction";
import { CipherType } from "@bitwarden/common/vault/enums/cipher-type";
import { Cipher } from "@bitwarden/common/vault/models/domain/cipher";
import { Folder } from "@bitwarden/common/vault/models/domain/folder";
import { Login } from "@bitwarden/common/vault/models/domain/login";
import { CipherView } from "@bitwarden/common/vault/models/view/cipher.view";
import { FolderView } from "@bitwarden/common/vault/models/view/folder.view";
import { LoginView } from "@bitwarden/common/vault/models/view/login.view";
import { BuildTestObject, GetUniqueString } from "../../../../common/spec/utils";
import { VaultExportService } from "./vault-export.service";
const UserCipherViews = [
generateCipherView(false),
generateCipherView(false),
generateCipherView(true),
];
const UserCipherDomains = [
generateCipherDomain(false),
generateCipherDomain(false),
generateCipherDomain(true),
];
const UserFolderViews = [generateFolderView(), generateFolderView()];
const UserFolders = [generateFolder(), generateFolder()];
function generateCipherView(deleted: boolean) {
return BuildTestObject(
{
id: GetUniqueString("id"),
notes: GetUniqueString("notes"),
type: CipherType.Login,
login: BuildTestObject<LoginView>(
{
username: GetUniqueString("username"),
password: GetUniqueString("password"),
},
LoginView
),
collectionIds: null,
deletedDate: deleted ? new Date() : null,
},
CipherView
);
}
function generateCipherDomain(deleted: boolean) {
return BuildTestObject(
{
id: GetUniqueString("id"),
notes: new EncString(GetUniqueString("notes")),
type: CipherType.Login,
login: BuildTestObject<Login>(
{
username: new EncString(GetUniqueString("username")),
password: new EncString(GetUniqueString("password")),
},
Login
),
collectionIds: null,
deletedDate: deleted ? new Date() : null,
},
Cipher
);
}
function generateFolderView() {
return BuildTestObject(
{
id: GetUniqueString("id"),
name: GetUniqueString("name"),
revisionDate: new Date(),
},
FolderView
);
}
function generateFolder() {
const actual = Folder.fromJSON({
revisionDate: new Date("2022-08-04T01:06:40.441Z").toISOString(),
name: "name",
id: "id",
});
return actual;
}
function expectEqualCiphers(ciphers: CipherView[] | Cipher[], jsonResult: string) {
const actual = JSON.stringify(JSON.parse(jsonResult).items);
const items: CipherWithIdExport[] = [];
ciphers.forEach((c: CipherView | Cipher) => {
const item = new CipherWithIdExport();
item.build(c);
items.push(item);
});
expect(actual).toEqual(JSON.stringify(items));
}
function expectEqualFolderViews(folderviews: FolderView[] | Folder[], jsonResult: string) {
const actual = JSON.stringify(JSON.parse(jsonResult).folders);
const folders: FolderResponse[] = [];
folderviews.forEach((c) => {
const folder = new FolderResponse();
folder.id = c.id;
folder.name = c.name.toString();
folders.push(folder);
});
expect(actual.length).toBeGreaterThan(0);
expect(actual).toEqual(JSON.stringify(folders));
}
function expectEqualFolders(folders: Folder[], jsonResult: string) {
const actual = JSON.stringify(JSON.parse(jsonResult).folders);
const items: Folder[] = [];
folders.forEach((c) => {
const item = new Folder();
item.id = c.id;
item.name = c.name;
items.push(item);
});
expect(actual.length).toBeGreaterThan(0);
expect(actual).toEqual(JSON.stringify(items));
}
describe("VaultExportService", () => {
let exportService: VaultExportService;
let apiService: SubstituteOf<ApiService>;
let cryptoFunctionService: SubstituteOf<CryptoFunctionService>;
let cipherService: SubstituteOf<CipherService>;
let folderService: SubstituteOf<FolderService>;
let cryptoService: SubstituteOf<CryptoService>;
let stateService: SubstituteOf<StateService>;
beforeEach(() => {
apiService = Substitute.for<ApiService>();
cryptoFunctionService = Substitute.for<CryptoFunctionService>();
cipherService = Substitute.for<CipherService>();
folderService = Substitute.for<FolderService>();
cryptoService = Substitute.for<CryptoService>();
stateService = Substitute.for<StateService>();
folderService.getAllDecryptedFromState().resolves(UserFolderViews);
folderService.getAllFromState().resolves(UserFolders);
stateService.getKdfType().resolves(KdfType.PBKDF2_SHA256);
stateService.getKdfConfig().resolves(new KdfConfig(DEFAULT_PBKDF2_ITERATIONS));
exportService = new VaultExportService(
folderService,
cipherService,
apiService,
cryptoService,
cryptoFunctionService,
stateService
);
});
it("exports unecrypted user ciphers", async () => {
cipherService.getAllDecrypted().resolves(UserCipherViews.slice(0, 1));
const actual = await exportService.getExport("json");
expectEqualCiphers(UserCipherViews.slice(0, 1), actual);
});
it("exports encrypted json user ciphers", async () => {
cipherService.getAll().resolves(UserCipherDomains.slice(0, 1));
const actual = await exportService.getExport("encrypted_json");
expectEqualCiphers(UserCipherDomains.slice(0, 1), actual);
});
it("does not unecrypted export trashed user items", async () => {
cipherService.getAllDecrypted().resolves(UserCipherViews);
const actual = await exportService.getExport("json");
expectEqualCiphers(UserCipherViews.slice(0, 2), actual);
});
it("does not encrypted export trashed user items", async () => {
cipherService.getAll().resolves(UserCipherDomains);
const actual = await exportService.getExport("encrypted_json");
expectEqualCiphers(UserCipherDomains.slice(0, 2), actual);
});
describe("password protected export", () => {
let exportString: string;
let exportObject: any;
let mac: SubstituteOf<EncString>;
let data: SubstituteOf<EncString>;
const password = "password";
const salt = "salt";
describe("export json object", () => {
beforeEach(async () => {
mac = Substitute.for<EncString>();
data = Substitute.for<EncString>();
mac.encryptedString.returns("mac");
data.encryptedString.returns("encData");
jest.spyOn(Utils, "fromBufferToB64").mockReturnValue(salt);
cipherService.getAllDecrypted().resolves(UserCipherViews.slice(0, 1));
exportString = await exportService.getPasswordProtectedExport(password);
exportObject = JSON.parse(exportString);
});
it("specifies it is encrypted", () => {
expect(exportObject.encrypted).toBe(true);
});
it("specifies it's password protected", () => {
expect(exportObject.passwordProtected).toBe(true);
});
it("specifies salt", () => {
expect(exportObject.salt).toEqual("salt");
});
it("specifies kdfIterations", () => {
expect(exportObject.kdfIterations).toEqual(DEFAULT_PBKDF2_ITERATIONS);
});
it("has kdfType", () => {
expect(exportObject.kdfType).toEqual(KdfType.PBKDF2_SHA256);
});
it("has a mac property", async () => {
cryptoService.encrypt(Arg.any(), Arg.any()).resolves(mac);
exportString = await exportService.getPasswordProtectedExport(password);
exportObject = JSON.parse(exportString);
expect(exportObject.encKeyValidation_DO_NOT_EDIT).toEqual(mac.encryptedString);
});
it("has data property", async () => {
cryptoService.encrypt(Arg.any(), Arg.any()).resolves(data);
exportString = await exportService.getPasswordProtectedExport(password);
exportObject = JSON.parse(exportString);
expect(exportObject.data).toEqual(data.encryptedString);
});
it("encrypts the data property", async () => {
const unencrypted = await exportService.getExport();
expect(exportObject.data).not.toEqual(unencrypted);
});
});
});
it("exported unencrypted object contains folders", async () => {
cipherService.getAllDecrypted().resolves(UserCipherViews.slice(0, 1));
await folderService.getAllDecryptedFromState();
const actual = await exportService.getExport("json");
expectEqualFolderViews(UserFolderViews, actual);
});
it("exported encrypted json contains folders", async () => {
cipherService.getAll().resolves(UserCipherDomains.slice(0, 1));
await folderService.getAllFromState();
const actual = await exportService.getExport("encrypted_json");
expectEqualFolders(UserFolders, actual);
});
});
export class FolderResponse {
id: string = null;
name: string = null;
}

View File

@@ -0,0 +1,419 @@
import * as papa from "papaparse";
import { ApiService } from "@bitwarden/common/abstractions/api.service";
import { CryptoService } from "@bitwarden/common/abstractions/crypto.service";
import { CryptoFunctionService } from "@bitwarden/common/abstractions/cryptoFunction.service";
import { StateService } from "@bitwarden/common/abstractions/state.service";
import { CollectionData } from "@bitwarden/common/admin-console/models/data/collection.data";
import { Collection } from "@bitwarden/common/admin-console/models/domain/collection";
import { CollectionDetailsResponse } from "@bitwarden/common/admin-console/models/response/collection.response";
import { CollectionView } from "@bitwarden/common/admin-console/models/view/collection.view";
import { KdfConfig } from "@bitwarden/common/auth/models/domain/kdf-config";
import { KdfType } from "@bitwarden/common/enums";
import { Utils } from "@bitwarden/common/misc/utils";
import {
CipherWithIdExport,
CollectionWithIdExport,
FolderWithIdExport,
} from "@bitwarden/common/models/export";
import { CipherService } from "@bitwarden/common/vault/abstractions/cipher.service";
import { FolderService } from "@bitwarden/common/vault/abstractions/folder/folder.service.abstraction";
import { CipherType } from "@bitwarden/common/vault/enums/cipher-type";
import { CipherData } from "@bitwarden/common/vault/models/data/cipher.data";
import { Cipher } from "@bitwarden/common/vault/models/domain/cipher";
import { Folder } from "@bitwarden/common/vault/models/domain/folder";
import { CipherView } from "@bitwarden/common/vault/models/view/cipher.view";
import { FolderView } from "@bitwarden/common/vault/models/view/folder.view";
import { ExportHelper } from "../../export-helper";
import { BitwardenPasswordProtectedFileFormat } from "../bitwarden-password-protected-types";
import { ExportFormat, VaultExportServiceAbstraction } from "./vault-export.service.abstraction";
export class VaultExportService implements VaultExportServiceAbstraction {
constructor(
private folderService: FolderService,
private cipherService: CipherService,
private apiService: ApiService,
private cryptoService: CryptoService,
private cryptoFunctionService: CryptoFunctionService,
private stateService: StateService
) {}
async getExport(format: ExportFormat = "csv", organizationId?: string): Promise<string> {
if (organizationId) {
return await this.getOrganizationExport(organizationId, format);
}
if (format === "encrypted_json") {
return this.getEncryptedExport();
} else {
return this.getDecryptedExport(format);
}
}
async getPasswordProtectedExport(password: string, organizationId?: string): Promise<string> {
const clearText = organizationId
? await this.getOrganizationExport(organizationId, "json")
: await this.getExport("json");
const kdfType: KdfType = await this.stateService.getKdfType();
const kdfConfig: KdfConfig = await this.stateService.getKdfConfig();
const salt = Utils.fromBufferToB64(await this.cryptoFunctionService.randomBytes(16));
const key = await this.cryptoService.makePinKey(password, salt, kdfType, kdfConfig);
const encKeyValidation = await this.cryptoService.encrypt(Utils.newGuid(), key);
const encText = await this.cryptoService.encrypt(clearText, key);
const jsonDoc: BitwardenPasswordProtectedFileFormat = {
encrypted: true,
passwordProtected: true,
salt: salt,
kdfType: kdfType,
kdfIterations: kdfConfig.iterations,
kdfMemory: kdfConfig.memory,
kdfParallelism: kdfConfig.parallelism,
encKeyValidation_DO_NOT_EDIT: encKeyValidation.encryptedString,
data: encText.encryptedString,
};
return JSON.stringify(jsonDoc, null, " ");
}
async getOrganizationExport(
organizationId: string,
format: ExportFormat = "csv"
): Promise<string> {
if (format === "encrypted_json") {
return this.getOrganizationEncryptedExport(organizationId);
} else {
return this.getOrganizationDecryptedExport(organizationId, format);
}
}
getFileName(prefix: string = null, extension = "csv"): string {
return ExportHelper.getFileName(prefix, extension);
}
private async getDecryptedExport(format: "json" | "csv"): Promise<string> {
let decFolders: FolderView[] = [];
let decCiphers: CipherView[] = [];
const promises = [];
promises.push(
this.folderService.getAllDecryptedFromState().then((folders) => {
decFolders = folders;
})
);
promises.push(
this.cipherService.getAllDecrypted().then((ciphers) => {
decCiphers = ciphers.filter((f) => f.deletedDate == null);
})
);
await Promise.all(promises);
if (format === "csv") {
const foldersMap = new Map<string, FolderView>();
decFolders.forEach((f) => {
if (f.id != null) {
foldersMap.set(f.id, f);
}
});
const exportCiphers: any[] = [];
decCiphers.forEach((c) => {
// only export logins and secure notes
if (c.type !== CipherType.Login && c.type !== CipherType.SecureNote) {
return;
}
if (c.organizationId != null) {
return;
}
const cipher: any = {};
cipher.folder =
c.folderId != null && foldersMap.has(c.folderId) ? foldersMap.get(c.folderId).name : null;
cipher.favorite = c.favorite ? 1 : null;
this.buildCommonCipher(cipher, c);
exportCiphers.push(cipher);
});
return papa.unparse(exportCiphers);
} else {
const jsonDoc: any = {
encrypted: false,
folders: [],
items: [],
};
decFolders.forEach((f) => {
if (f.id == null) {
return;
}
const folder = new FolderWithIdExport();
folder.build(f);
jsonDoc.folders.push(folder);
});
decCiphers.forEach((c) => {
if (c.organizationId != null) {
return;
}
const cipher = new CipherWithIdExport();
cipher.build(c);
cipher.collectionIds = null;
jsonDoc.items.push(cipher);
});
return JSON.stringify(jsonDoc, null, " ");
}
}
private async getEncryptedExport(): Promise<string> {
let folders: Folder[] = [];
let ciphers: Cipher[] = [];
const promises = [];
promises.push(
this.folderService.getAllFromState().then((f) => {
folders = f;
})
);
promises.push(
this.cipherService.getAll().then((c) => {
ciphers = c.filter((f) => f.deletedDate == null);
})
);
await Promise.all(promises);
const encKeyValidation = await this.cryptoService.encrypt(Utils.newGuid());
const jsonDoc: any = {
encrypted: true,
encKeyValidation_DO_NOT_EDIT: encKeyValidation.encryptedString,
folders: [],
items: [],
};
folders.forEach((f) => {
if (f.id == null) {
return;
}
const folder = new FolderWithIdExport();
folder.build(f);
jsonDoc.folders.push(folder);
});
ciphers.forEach((c) => {
if (c.organizationId != null) {
return;
}
const cipher = new CipherWithIdExport();
cipher.build(c);
cipher.collectionIds = null;
jsonDoc.items.push(cipher);
});
return JSON.stringify(jsonDoc, null, " ");
}
private async getOrganizationDecryptedExport(
organizationId: string,
format: "json" | "csv"
): Promise<string> {
const decCollections: CollectionView[] = [];
const decCiphers: CipherView[] = [];
const promises = [];
promises.push(
this.apiService.getOrganizationExport(organizationId).then((exportData) => {
const exportPromises: any = [];
if (exportData != null) {
if (exportData.collections != null && exportData.collections.length > 0) {
exportData.collections.forEach((c) => {
const collection = new Collection(new CollectionData(c as CollectionDetailsResponse));
exportPromises.push(
collection.decrypt().then((decCol) => {
decCollections.push(decCol);
})
);
});
}
if (exportData.ciphers != null && exportData.ciphers.length > 0) {
exportData.ciphers
.filter((c) => c.deletedDate === null)
.forEach((c) => {
const cipher = new Cipher(new CipherData(c));
exportPromises.push(
cipher.decrypt().then((decCipher) => {
decCiphers.push(decCipher);
})
);
});
}
}
return Promise.all(exportPromises);
})
);
await Promise.all(promises);
if (format === "csv") {
const collectionsMap = new Map<string, CollectionView>();
decCollections.forEach((c) => {
collectionsMap.set(c.id, c);
});
const exportCiphers: any[] = [];
decCiphers.forEach((c) => {
// only export logins and secure notes
if (c.type !== CipherType.Login && c.type !== CipherType.SecureNote) {
return;
}
const cipher: any = {};
cipher.collections = [];
if (c.collectionIds != null) {
cipher.collections = c.collectionIds
.filter((id) => collectionsMap.has(id))
.map((id) => collectionsMap.get(id).name);
}
this.buildCommonCipher(cipher, c);
exportCiphers.push(cipher);
});
return papa.unparse(exportCiphers);
} else {
const jsonDoc: any = {
encrypted: false,
collections: [],
items: [],
};
decCollections.forEach((c) => {
const collection = new CollectionWithIdExport();
collection.build(c);
jsonDoc.collections.push(collection);
});
decCiphers.forEach((c) => {
const cipher = new CipherWithIdExport();
cipher.build(c);
jsonDoc.items.push(cipher);
});
return JSON.stringify(jsonDoc, null, " ");
}
}
private async getOrganizationEncryptedExport(organizationId: string): Promise<string> {
const collections: Collection[] = [];
const ciphers: Cipher[] = [];
const promises = [];
promises.push(
this.apiService.getCollections(organizationId).then((c) => {
const collectionPromises: any = [];
if (c != null && c.data != null && c.data.length > 0) {
c.data.forEach((r) => {
const collection = new Collection(new CollectionData(r as CollectionDetailsResponse));
collections.push(collection);
});
}
return Promise.all(collectionPromises);
})
);
promises.push(
this.apiService.getCiphersOrganization(organizationId).then((c) => {
const cipherPromises: any = [];
if (c != null && c.data != null && c.data.length > 0) {
c.data
.filter((item) => item.deletedDate === null)
.forEach((item) => {
const cipher = new Cipher(new CipherData(item));
ciphers.push(cipher);
});
}
return Promise.all(cipherPromises);
})
);
await Promise.all(promises);
const orgKey = await this.cryptoService.getOrgKey(organizationId);
const encKeyValidation = await this.cryptoService.encrypt(Utils.newGuid(), orgKey);
const jsonDoc: any = {
encrypted: true,
encKeyValidation_DO_NOT_EDIT: encKeyValidation.encryptedString,
collections: [],
items: [],
};
collections.forEach((c) => {
const collection = new CollectionWithIdExport();
collection.build(c);
jsonDoc.collections.push(collection);
});
ciphers.forEach((c) => {
const cipher = new CipherWithIdExport();
cipher.build(c);
jsonDoc.items.push(cipher);
});
return JSON.stringify(jsonDoc, null, " ");
}
private buildCommonCipher(cipher: any, c: CipherView) {
cipher.type = null;
cipher.name = c.name;
cipher.notes = c.notes;
cipher.fields = null;
cipher.reprompt = c.reprompt;
// Login props
cipher.login_uri = null;
cipher.login_username = null;
cipher.login_password = null;
cipher.login_totp = null;
if (c.fields) {
c.fields.forEach((f: any) => {
if (!cipher.fields) {
cipher.fields = "";
} else {
cipher.fields += "\n";
}
cipher.fields += (f.name || "") + ": " + f.value;
});
}
switch (c.type) {
case CipherType.Login:
cipher.type = "login";
cipher.login_username = c.login.username;
cipher.login_password = c.login.password;
cipher.login_totp = c.login.totp;
if (c.login.uris) {
cipher.login_uri = [];
c.login.uris.forEach((u) => {
cipher.login_uri.push(u.uri);
});
}
break;
case CipherType.SecureNote:
cipher.type = "note";
break;
default:
return;
}
return cipher;
}
}