From fb122cbbdbc45168b497675b39c49a5882dbef39 Mon Sep 17 00:00:00 2001 From: Kyle Spearrin Date: Thu, 2 Jul 2020 14:50:54 -0400 Subject: [PATCH] fix okta paging (#51) * fix okta paging * remove okta package * use node https instead of a library * remove bent types * add 500ms throttle to avoid rate limiter --- package-lock.json | 37 +------ package.json | 1 - src/services/okta-directory.service.ts | 141 ++++++++++++++++++++----- 3 files changed, 117 insertions(+), 62 deletions(-) diff --git a/package-lock.json b/package-lock.json index ab826377..5c9e045b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -767,19 +767,6 @@ "integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==", "dev": true }, - "@okta/okta-sdk-nodejs": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@okta/okta-sdk-nodejs/-/okta-sdk-nodejs-2.0.1.tgz", - "integrity": "sha512-3TSvzipQvkzLMwS3fzHd8qqti9UeabeICRPF6HBLzgNq5exrlXdYMpqYEdNbj0hwjhxrNdGEOZXTqznqrinWiQ==", - "requires": { - "deep-copy": "^1.4.2", - "flat": "^2.0.1", - "isomorphic-fetch": "2.2.1", - "js-yaml": "^3.13.0", - "lodash": "^4.17.14", - "parse-link-header": "1.0.0" - } - }, "@sindresorhus/is": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", @@ -4378,11 +4365,6 @@ "mimic-response": "^1.0.0" } }, - "deep-copy": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/deep-copy/-/deep-copy-1.4.2.tgz", - "integrity": "sha512-VxZwQ/1+WGQPl5nE67uLhh7OqdrmqI1OazrraO9Bbw/M8Bt6Mol/RxzDA6N6ZgRXpsG/W9PgUj8E1LHHBEq2GQ==" - }, "deep-extend": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", @@ -7032,14 +7014,6 @@ "integrity": "sha1-Tnmumy6zi/hrO7Vr8+ClaqX8q9c=", "dev": true }, - "flat": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/flat/-/flat-2.0.1.tgz", - "integrity": "sha1-cOKRiKdL4MPIlAnu0fqVd5B64y8=", - "requires": { - "is-buffer": "~1.1.2" - } - }, "flush-write-stream": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.0.3.tgz", @@ -9871,7 +9845,8 @@ "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dev": true }, "is-builtin-module": { "version": "1.0.0", @@ -12358,14 +12333,6 @@ "error-ex": "^1.2.0" } }, - "parse-link-header": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/parse-link-header/-/parse-link-header-1.0.0.tgz", - "integrity": "sha1-34N7t0vowC55MYAUVo+qidw05jc=", - "requires": { - "xtend": "~4.0.0" - } - }, "parse-passwd": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/parse-passwd/-/parse-passwd-1.0.0.tgz", diff --git a/package.json b/package.json index 0c9bcf7c..a0f0a929 100644 --- a/package.json +++ b/package.json @@ -199,7 +199,6 @@ "@angular/router": "7.2.1", "@angular/upgrade": "7.2.1", "@microsoft/microsoft-graph-client": "1.2.0", - "@okta/okta-sdk-nodejs": "2.0.1", "angular2-toaster": "6.1.0", "angulartics2": "6.3.0", "big-integer": "1.6.36", diff --git a/src/services/okta-directory.service.ts b/src/services/okta-directory.service.ts index c3fde588..a648f428 100644 --- a/src/services/okta-directory.service.ts +++ b/src/services/okta-directory.service.ts @@ -12,13 +12,11 @@ import { DirectoryService } from './directory.service'; import { I18nService } from 'jslib/abstractions/i18n.service'; import { LogService } from 'jslib/abstractions/log.service'; -// tslint:disable-next-line -const okta = require('@okta/okta-sdk-nodejs'); +import * as https from 'https'; export class OktaDirectoryService extends BaseDirectoryService implements DirectoryService { private dirConfig: OktaConfiguration; private syncConfig: SyncConfiguration; - private client: any; constructor(private configurationService: ConfigurationService, private logService: LogService, private i18nService: I18nService) { @@ -45,11 +43,6 @@ export class OktaDirectoryService extends BaseDirectoryService implements Direct throw new Error(this.i18nService.t('dirConfigIncomplete')); } - this.client = new okta.Client({ - orgUrl: this.dirConfig.orgUrl, - token: this.dirConfig.token, - }); - let users: UserEntry[]; if (this.syncConfig.users) { users = await this.getUsers(force); @@ -72,12 +65,15 @@ export class OktaDirectoryService extends BaseDirectoryService implements Direct const setFilter = this.createCustomSet(this.syncConfig.userFilter); this.logService.info('Querying users.'); - const usersPromise = this.client.listUsers({ filter: oktaFilter }).each((user: any) => { - const entry = this.buildUser(user); - if (entry != null && !this.filterOutResult(setFilter, entry.email)) { - entries.push(entry); - } - }); + const usersPromise = this.apiGetMany('users?filter=' + this.encodeUrlParameter(oktaFilter)) + .then((users: any[]) => { + for (const user of users) { + const entry = this.buildUser(user); + if (entry != null && !this.filterOutResult(setFilter, entry.email)) { + entries.push(entry); + } + } + }); // Deactivated users have to be queried for separately, only when no filter is provided in the first query let deactUsersPromise: any; @@ -86,12 +82,15 @@ export class OktaDirectoryService extends BaseDirectoryService implements Direct if (oktaFilter != null) { deactOktaFilter = '(' + oktaFilter + ') and ' + deactOktaFilter; } - deactUsersPromise = this.client.listUsers({ filter: deactOktaFilter }).each((user: any) => { - const entry = this.buildUser(user); - if (entry != null && !this.filterOutResult(setFilter, entry.email)) { - entries.push(entry); - } - }); + deactUsersPromise = this.apiGetMany('users?filter=' + this.encodeUrlParameter(deactOktaFilter)) + .then((users: any[]) => { + for (const user of users) { + const entry = this.buildUser(user); + if (entry != null && !this.filterOutResult(setFilter, entry.email)) { + entries.push(entry); + } + } + }); } else { deactUsersPromise = Promise.resolve(); } @@ -116,10 +115,14 @@ export class OktaDirectoryService extends BaseDirectoryService implements Direct const oktaFilter = this.buildOktaFilter(this.syncConfig.groupFilter, force, lastSync); this.logService.info('Querying groups.'); - await this.client.listGroups({ filter: oktaFilter }).each(async (group: any) => { - const entry = await this.buildGroup(group); - if (entry != null && !this.filterOutResult(setFilter, entry.name)) { - entries.push(entry); + await this.apiGetMany('groups?filter=' + this.encodeUrlParameter(oktaFilter)).then(async (groups: any[]) => { + for (const group of groups) { + const entry = await this.buildGroup(group); + if (entry != null && !this.filterOutResult(setFilter, entry.name)) { + entries.push(entry); + } + // throttle some to avoid rate limiting + await new Promise((resolve) => setTimeout(resolve, 500)); } }); return entries; @@ -131,8 +134,10 @@ export class OktaDirectoryService extends BaseDirectoryService implements Direct entry.referenceId = group.id; entry.name = group.profile.name; - await this.client.listGroupUsers(group.id).each((user: any) => { - entry.userMemberExternalIds.add(user.id); + await this.apiGetMany('groups/' + group.id + '/users').then((users: any[]) => { + for (const user of users) { + entry.userMemberExternalIds.add(user.id); + } }); return entry; @@ -152,4 +157,88 @@ export class OktaDirectoryService extends BaseDirectoryService implements Direct return '(' + baseFilter + ') and ' + updatedFilter; } + + private encodeUrlParameter(filter: string): string { + return filter == null ? '' : encodeURIComponent(filter); + } + + private async apiGetCall(url: string): Promise<[any, Map]> { + const u = new URL(url); + return new Promise((resolve) => { + https.get({ + hostname: u.hostname, + path: u.pathname + u.search, + port: 443, + headers: { + Authorization: 'SSWS ' + this.dirConfig.token, + Accept: 'application/json', + }, + }, (res) => { + let body = ''; + + res.on('data', (chunk) => { + body += chunk; + }); + + res.on('end', () => { + if (res.statusCode !== 200) { + resolve(null); + return; + } + + const responseJson = JSON.parse(body); + if (res.headers != null) { + const headersMap = new Map(); + for (const key in res.headers) { + if (res.headers.hasOwnProperty(key)) { + const val = res.headers[key]; + headersMap.set(key.toLowerCase(), val); + } + } + resolve([responseJson, headersMap]); + return; + } + resolve([responseJson, null]); + }); + + res.on('error', () => { + resolve(null); + }); + }); + }); + } + + private async apiGetMany(endpoint: string, currentData: any[] = []): Promise { + const url = endpoint.indexOf('https://') === 0 ? endpoint : `${this.dirConfig.orgUrl}/api/v1/${endpoint}`; + const response = await this.apiGetCall(url); + if (response == null || response[0] == null || !Array.isArray(response[0])) { + throw new Error('API call failed.'); + } + if (response[0].length === 0) { + return currentData; + } + currentData = currentData.concat(response[0]); + if (response[1] == null) { + return currentData; + } + const linkHeader = response[1].get('link'); + if (linkHeader == null || Array.isArray(linkHeader)) { + return currentData; + } + let nextLink: string = null; + const linkHeaderParts = linkHeader.split(','); + for (const part of linkHeaderParts) { + if (part.indexOf('; rel="next"') > -1) { + const subParts = part.split(';'); + if (subParts.length > 0 && subParts[0].indexOf('https://') > -1) { + nextLink = subParts[0].replace('>', '').replace('<', '').trim(); + break; + } + } + } + if (nextLink == null) { + return currentData; + } + return this.apiGetMany(nextLink, currentData); + } }