From 95d98181fc3b51b47dcc9bed01046fd9f3355e55 Mon Sep 17 00:00:00 2001 From: Kyle Spearrin Date: Wed, 2 May 2018 22:00:44 -0400 Subject: [PATCH] sync logic --- src/app/services/services.module.ts | 2 +- src/app/tabs/dashboard.component.ts | 16 +++--- src/services/azure-directory.service.ts | 27 +++++----- src/services/configuration.service.ts | 26 ++++++++++ src/services/directory.service.ts | 2 +- src/services/gsuite-directory.service.ts | 18 +++---- src/services/ldap-directory.service.ts | 19 ++++--- src/services/sync.service.ts | 63 +++++++++++++++++------- 8 files changed, 114 insertions(+), 59 deletions(-) diff --git a/src/app/services/services.module.ts b/src/app/services/services.module.ts index 748db623..41be3796 100644 --- a/src/app/services/services.module.ts +++ b/src/app/services/services.module.ts @@ -71,7 +71,7 @@ const containerService = new ContainerService(cryptoService, platformUtilsServic const authService = new AuthService(cryptoService, apiService, userService, tokenService, appIdService, i18nService, platformUtilsService, messagingService, false); const configurationService = new ConfigurationService(storageService, secureStorageService); -const syncSevrice = new SyncService(configurationService, logService); +const syncSevrice = new SyncService(configurationService, logService, cryptoFunctionService, apiService); const analytics = new Analytics(window, () => true, platformUtilsService, storageService, appIdService); containerService.attachToWindow(window); diff --git a/src/app/tabs/dashboard.component.ts b/src/app/tabs/dashboard.component.ts index 83cde573..276a82b7 100644 --- a/src/app/tabs/dashboard.component.ts +++ b/src/app/tabs/dashboard.component.ts @@ -28,7 +28,7 @@ export class DashboardComponent { constructor(private i18nService: I18nService, private syncService: SyncService) { } async sync() { - this.syncPromise = this.syncService.sync(false, true); + this.syncPromise = this.syncService.sync(false, false); await this.syncPromise; } @@ -41,7 +41,7 @@ export class DashboardComponent { this.simPromise = new Promise(async (resolve, reject) => { try { - const result = await this.syncService.sync(!this.simSinceLast, false); + const result = await this.syncService.sync(!this.simSinceLast, true); this.simUsers = result[1]; this.simGroups = result[0]; } catch (e) { @@ -51,7 +51,7 @@ export class DashboardComponent { const userMap = new Map(); if (this.simUsers != null) { this.sort(this.simUsers); - this.simUsers.forEach((u) => { + for (const u of this.simUsers) { userMap.set(u.externalId, u); if (u.deleted) { this.simDeletedUsers.push(u); @@ -60,29 +60,29 @@ export class DashboardComponent { } else { this.simEnabledUsers.push(u); } - }); + } } if (userMap.size > 0 && this.simGroups != null) { this.sort(this.simGroups); - this.simGroups.forEach((g) => { + for (const g of this.simGroups) { if (g.userMemberExternalIds == null) { return; } - g.userMemberExternalIds.forEach((uid) => { + for (const uid of g.userMemberExternalIds) { if (userMap.has(uid)) { if ((g as any).users == null) { (g as any).users = []; } (g as any).users.push(userMap.get(uid)); } - }); + } if ((g as any).users != null) { this.sort((g as any).users); } - }); + } } resolve(); }); diff --git a/src/services/azure-directory.service.ts b/src/services/azure-directory.service.ts index 4cfa30e8..aacb784f 100644 --- a/src/services/azure-directory.service.ts +++ b/src/services/azure-directory.service.ts @@ -28,7 +28,7 @@ export class AzureDirectoryService implements DirectoryService { this.init(); } - async getEntries(force = false): Promise<[GroupEntry[], UserEntry[]]> { + async getEntries(force: boolean, test: boolean): Promise<[GroupEntry[], UserEntry[]]> { const type = await this.configurationService.getDirectoryType(); if (type !== DirectoryType.AzureActiveDirectory) { return; @@ -47,7 +47,7 @@ export class AzureDirectoryService implements DirectoryService { let users: UserEntry[]; if (this.syncConfig.users) { - users = await this.getUsers(force); + users = await this.getUsers(force, !test); } let groups: GroupEntry[]; @@ -57,7 +57,7 @@ export class AzureDirectoryService implements DirectoryService { const groupForce = force || (users != null && users.filter((u) => !u.deleted && !u.disabled).length > 0); - groups = await this.getGroups(groupForce, setFilter); + groups = await this.getGroups(groupForce, !test, setFilter); if (setFilter != null && users != null) { users = users.filter((u) => { if (u.disabled || u.deleted) { @@ -72,7 +72,7 @@ export class AzureDirectoryService implements DirectoryService { return [groups, users]; } - private async getUsers(force: boolean): Promise { + private async getUsers(force: boolean, saveDelta: boolean): Promise { const entries: UserEntry[] = []; let res: any = null; @@ -95,7 +95,7 @@ export class AzureDirectoryService implements DirectoryService { while (true) { const users: graphType.User[] = res.value; if (users != null) { - users.forEach((user) => { + for (const user of users) { const entry = this.buildUser(user); if (this.filterOutResult(filter, entry.email)) { return; @@ -107,11 +107,11 @@ export class AzureDirectoryService implements DirectoryService { } entries.push(entry); - }); + } } if (res[NextLink] == null) { - if (res[DeltaLink] != null) { + if (res[DeltaLink] != null && saveDelta) { await this.configurationService.saveUserDeltaToken(res[DeltaLink]); } break; @@ -138,7 +138,8 @@ export class AzureDirectoryService implements DirectoryService { return entry; } - private async getGroups(force: boolean, setFilter: [boolean, Set]): Promise { + private async getGroups(force: boolean, saveDelta: boolean, + setFilter: [boolean, Set]): Promise { const entries: GroupEntry[] = []; const changedGroupIds: string[] = []; const token = await this.configurationService.getGroupDeltaToken(); @@ -179,7 +180,7 @@ export class AzureDirectoryService implements DirectoryService { } if (res[NextLink] == null) { - if (res[DeltaLink] != null) { + if (res[DeltaLink] != null && saveDelta) { await this.configurationService.saveGroupDeltaToken(res[DeltaLink]); } break; @@ -232,13 +233,13 @@ export class AzureDirectoryService implements DirectoryService { const memRes = await memReq.get(); const members: any = memRes.value; if (members != null) { - members.forEach((member: any) => { + for (const member of members) { if (member[ObjectType] === '#microsoft.graph.group') { entry.groupMemberReferenceIds.add((member as graphType.Group).id); } else if (member[ObjectType] === '#microsoft.graph.user') { entry.userMemberExternalIds.add((member as graphType.User).id); } - }); + } } return entry; @@ -266,9 +267,9 @@ export class AzureDirectoryService implements DirectoryService { const set = new Set(); const pieces = parts[1].split(','); - pieces.forEach((p) => { + for (const p of pieces) { set.add(p.trim().toLowerCase()); - }); + } return [exclude, set]; } diff --git a/src/services/configuration.service.ts b/src/services/configuration.service.ts index 991062c8..39836229 100644 --- a/src/services/configuration.service.ts +++ b/src/services/configuration.service.ts @@ -18,6 +18,8 @@ const Keys = { groupDelta: 'groupDeltaToken', lastUserSync: 'lastUserSync', lastGroupSync: 'lastGroupSync', + lastSyncHash: 'lastSyncHash', + organizationId: 'organizationId', }; export class ConfigurationService { @@ -145,4 +147,28 @@ export class ConfigurationService { return this.storageService.save(Keys.lastGroupSync, date); } } + + getLastSyncHash(): Promise { + return this.storageService.get(Keys.lastSyncHash); + } + + saveLastSyncHash(hash: string) { + if (hash == null) { + return this.storageService.remove(Keys.lastSyncHash); + } else { + return this.storageService.save(Keys.lastSyncHash, hash); + } + } + + getOrganizationId(): Promise { + return this.storageService.get(Keys.organizationId); + } + + saveOrganizationId(id: string) { + if (id == null) { + return this.storageService.remove(Keys.organizationId); + } else { + return this.storageService.save(Keys.organizationId, id); + } + } } diff --git a/src/services/directory.service.ts b/src/services/directory.service.ts index 72bcaf4b..88b0ac81 100644 --- a/src/services/directory.service.ts +++ b/src/services/directory.service.ts @@ -2,5 +2,5 @@ import { GroupEntry } from '../models/groupEntry'; import { UserEntry } from '../models/userEntry'; export interface DirectoryService { - getEntries(force?: boolean): Promise<[GroupEntry[], UserEntry[]]>; + getEntries(force: boolean, test: boolean): Promise<[GroupEntry[], UserEntry[]]>; } diff --git a/src/services/gsuite-directory.service.ts b/src/services/gsuite-directory.service.ts index 940550f2..481e8f96 100644 --- a/src/services/gsuite-directory.service.ts +++ b/src/services/gsuite-directory.service.ts @@ -32,7 +32,7 @@ export class GSuiteDirectoryService implements DirectoryService { this.service = google.admin('directory_v1'); } - async getEntries(force = false): Promise<[GroupEntry[], UserEntry[]]> { + async getEntries(force: boolean, test: boolean): Promise<[GroupEntry[], UserEntry[]]> { const type = await this.configurationService.getDirectoryType(); if (type !== DirectoryType.GSuite) { return; @@ -76,7 +76,7 @@ export class GSuiteDirectoryService implements DirectoryService { const filter = this.createSet(this.syncConfig.userFilter); if (res.data.users != null) { - res.data.users.forEach((user) => { + for (const user of res.data.users) { if (this.filterOutResult(filter, user.primaryEmail)) { return; } @@ -85,7 +85,7 @@ export class GSuiteDirectoryService implements DirectoryService { if (entry != null) { entries.push(entry); } - }); + } } this.logService.info('Querying deleted users.'); @@ -96,7 +96,7 @@ export class GSuiteDirectoryService implements DirectoryService { } if (delRes.data.users != null) { - delRes.data.users.forEach((user) => { + for (const user of delRes.data.users) { if (this.filterOutResult(filter, user.primaryEmail)) { return; } @@ -105,7 +105,7 @@ export class GSuiteDirectoryService implements DirectoryService { if (entry != null) { entries.push(entry); } - }); + } } return entries; @@ -164,7 +164,7 @@ export class GSuiteDirectoryService implements DirectoryService { } if (memRes.data.members != null) { - memRes.data.members.forEach((member) => { + for (const member of memRes.data.members) { if (member.role.toLowerCase() !== 'member') { return; } @@ -177,7 +177,7 @@ export class GSuiteDirectoryService implements DirectoryService { } else if (member.type.toLowerCase() === 'group') { entry.groupMemberReferenceIds.add(member.id); } - }); + } } return entry; @@ -223,9 +223,9 @@ export class GSuiteDirectoryService implements DirectoryService { const set = new Set(); const pieces = parts[1].split(','); - pieces.forEach((p) => { + for (const p of pieces) { set.add(p.trim().toLowerCase()); - }); + } return [exclude, set]; } diff --git a/src/services/ldap-directory.service.ts b/src/services/ldap-directory.service.ts index 6fdf1a74..7f829d7c 100644 --- a/src/services/ldap-directory.service.ts +++ b/src/services/ldap-directory.service.ts @@ -22,7 +22,7 @@ export class LdapDirectoryService implements DirectoryService { constructor(private configurationService: ConfigurationService, private logService: LogService) { } - async getEntries(force = false): Promise<[GroupEntry[], UserEntry[]]> { + async getEntries(force: boolean, test: boolean): Promise<[GroupEntry[], UserEntry[]]> { const type = await this.configurationService.getDirectoryType(); if (type !== DirectoryType.Ldap) { return; @@ -61,7 +61,6 @@ export class LdapDirectoryService implements DirectoryService { private async getUsers(force: boolean): Promise { const lastSync = await this.configurationService.getLastUserSyncDate(); - let filter = this.buildBaseFilter(this.syncConfig.userObjectClass, this.syncConfig.userFilter); filter = this.buildRevisionFilter(filter, force, lastSync); @@ -121,7 +120,6 @@ export class LdapDirectoryService implements DirectoryService { const entries: GroupEntry[] = []; const lastSync = await this.configurationService.getLastUserSyncDate(); - const originalFilter = this.buildBaseFilter(this.syncConfig.groupObjectClass, this.syncConfig.groupFilter); let filter = originalFilter; const revisionFilter = this.buildRevisionFilter(filter, force, lastSync); @@ -151,12 +149,12 @@ export class LdapDirectoryService implements DirectoryService { return se; }); - groupSearchEntries.forEach((se) => { + for (const se of groupSearchEntries) { const group = this.buildGroup(se, userIdMap); if (group != null) { entries.push(group); } - }); + } return entries; } @@ -181,13 +179,13 @@ export class LdapDirectoryService implements DirectoryService { const members = this.getAttrVals(searchEntry, this.syncConfig.memberAttribute); if (members != null) { - members.forEach((memDn) => { + for (const memDn of members) { if (userMap.has(memDn) && !group.userMemberExternalIds.has(userMap.get(memDn))) { group.userMemberExternalIds.add(userMap.get(memDn)); } else if (!group.groupMemberReferenceIds.has(memDn)) { group.groupMemberReferenceIds.add(memDn); } - }); + } } return group; @@ -326,9 +324,14 @@ export class LdapDirectoryService implements DirectoryService { const pass = this.dirConfig.password == null || this.dirConfig.password.trim() === '' ? null : this.dirConfig.password; + if (user == null || pass == null) { + reject('Username and/or password re not configured.'); + return; + } + this.client.bind(user, pass, (err) => { if (err != null) { - reject(err); + reject('Error authenticating: ' + err.message); } else { resolve(); } diff --git a/src/services/sync.service.ts b/src/services/sync.service.ts index 12de3d73..d8bcb7b5 100644 --- a/src/services/sync.service.ts +++ b/src/services/sync.service.ts @@ -7,6 +7,8 @@ import { ImportDirectoryRequest } from 'jslib/models/request/importDirectoryRequ import { ImportDirectoryRequestGroup } from 'jslib/models/request/importDirectoryRequestGroup'; import { ImportDirectoryRequestUser } from 'jslib/models/request/importDirectoryRequestUser'; +import { ApiService } from 'jslib/abstractions/api.service'; +import { CryptoFunctionService } from 'jslib/abstractions/cryptoFunction.service'; import { LogService } from 'jslib/abstractions/log.service'; import { StorageService } from 'jslib/abstractions/storage.service'; @@ -15,6 +17,7 @@ import { ConfigurationService } from './configuration.service'; import { DirectoryService } from './directory.service'; import { GSuiteDirectoryService } from './gsuite-directory.service'; import { LdapDirectoryService } from './ldap-directory.service'; +import { Utils } from 'jslib/misc/utils'; const Keys = { }; @@ -22,9 +25,10 @@ const Keys = { export class SyncService { private dirType: DirectoryType; - constructor(private configurationService: ConfigurationService, private logService: LogService) { } + constructor(private configurationService: ConfigurationService, private logService: LogService, + private cryptoFunctionService: CryptoFunctionService, private apiService: ApiService) { } - async sync(force = true, sendToServer = true): Promise<[GroupEntry[], UserEntry[]]> { + async sync(force: boolean, test: boolean): Promise<[GroupEntry[], UserEntry[]]> { this.dirType = await this.configurationService.getDirectoryType(); if (this.dirType == null) { throw new Error('No directory configured.'); @@ -41,7 +45,7 @@ export class SyncService { const now = new Date(); try { - const entries = await directoryService.getEntries(force); + const entries = await directoryService.getEntries(force, test); const groups = entries[0]; const users = entries[1]; @@ -49,27 +53,48 @@ export class SyncService { this.flattenUsersToGroups(groups, null, groups); } - console.log(groups); - console.log(users); - - if (!sendToServer) { - // TODO: restore deltas - } - - if (!sendToServer || groups == null || groups.length === 0 || users == null || users.length === 0) { + if (test || groups == null || groups.length === 0 || users == null || users.length === 0) { return [groups, users]; } const req = this.buildRequest(groups, users, syncConfig.removeDisabled); + const reqJson = JSON.stringify(req); + + let hash: string = null; + const hashBuf = await this.cryptoFunctionService.hash(this.apiService.baseUrl + reqJson, 'sha256'); + if (hashBuf != null) { + hash = Utils.fromBufferToB64(hashBuf); + } + const lastHash = await this.configurationService.getLastSyncHash(); + + if (lastHash == null || hash !== lastHash) { + const orgId = await this.configurationService.getOrganizationId(); + if (orgId == null) { + throw new Error('Organization not set.'); + } + + const res = await this.apiService.postImportDirectory(orgId, req); + await this.configurationService.saveLastSyncHash(hash); + if (syncConfig.groups) { + await this.configurationService.saveLastGroupSyncDate(now); + } + if (syncConfig.users) { + await this.configurationService.saveLastUserSyncDate(now); + } + } + + return [groups, users]; } catch (e) { - // TODO: restore deltas - // failed sync result + if (!test) { + await this.configurationService.saveGroupDeltaToken(startingGroupDelta); + await this.configurationService.saveUserDeltaToken(startingUserDelta); + } throw e; } } private flattenUsersToGroups(currentGroups: GroupEntry[], currentGroupsUsers: string[], allGroups: GroupEntry[]) { - currentGroups.forEach((group) => { + for (const group of currentGroups) { const groupsInThisGroup = allGroups.filter((g) => group.groupMemberReferenceIds.has(g.referenceId)); let usersInThisGroup = Array.from(group.userMemberExternalIds); @@ -79,7 +104,7 @@ export class SyncService { } this.flattenUsersToGroups(groupsInThisGroup, usersInThisGroup, allGroups); - }); + } } private getDirectoryService(): DirectoryService { @@ -99,23 +124,23 @@ export class SyncService { const model = new ImportDirectoryRequest(); if (groups != null) { - groups.forEach((g) => { + for (const g of groups) { const ig = new ImportDirectoryRequestGroup(); ig.name = g.name; ig.externalId = g.externalId; ig.users = Array.from(g.userMemberExternalIds); model.groups.push(ig); - }); + } } if (users != null) { - users.forEach((u) => { + for (const u of users) { const iu = new ImportDirectoryRequestUser(); iu.email = u.email; iu.externalId = u.externalId; iu.deleted = u.deleted || (removeDisabled && u.disabled); model.users.push(iu); - }); + } } return model;