fix okta paging (#51)

* fix okta paging

* remove okta package

* use node https instead of a library

* remove bent types

* add 500ms throttle to avoid rate limiter

src/services/okta-directory.service.ts

@@ -12,13 +12,11 @@ import { DirectoryService } from './directory.service';
 import { I18nService } from 'jslib/abstractions/i18n.service';
 import { LogService } from 'jslib/abstractions/log.service';
 
-// tslint:disable-next-line
-const okta = require('@okta/okta-sdk-nodejs');
+import * as https from 'https';
 
 export class OktaDirectoryService extends BaseDirectoryService implements DirectoryService {
     private dirConfig: OktaConfiguration;
     private syncConfig: SyncConfiguration;
-    private client: any;
 
     constructor(private configurationService: ConfigurationService, private logService: LogService,
         private i18nService: I18nService) {
@@ -45,11 +43,6 @@ export class OktaDirectoryService extends BaseDirectoryService implements Direct
             throw new Error(this.i18nService.t('dirConfigIncomplete'));
         }
 
-        this.client = new okta.Client({
-            orgUrl: this.dirConfig.orgUrl,
-            token: this.dirConfig.token,
-        });
-
         let users: UserEntry[];
         if (this.syncConfig.users) {
             users = await this.getUsers(force);
@@ -72,12 +65,15 @@ export class OktaDirectoryService extends BaseDirectoryService implements Direct
         const setFilter = this.createCustomSet(this.syncConfig.userFilter);
 
         this.logService.info('Querying users.');
-        const usersPromise = this.client.listUsers({ filter: oktaFilter }).each((user: any) => {
-            const entry = this.buildUser(user);
-            if (entry != null && !this.filterOutResult(setFilter, entry.email)) {
-                entries.push(entry);
-            }
-        });
+        const usersPromise = this.apiGetMany('users?filter=' + this.encodeUrlParameter(oktaFilter))
+            .then((users: any[]) => {
+                for (const user of users) {
+                    const entry = this.buildUser(user);
+                    if (entry != null && !this.filterOutResult(setFilter, entry.email)) {
+                        entries.push(entry);
+                    }
+                }
+            });
 
         // Deactivated users have to be queried for separately, only when no filter is provided in the first query
         let deactUsersPromise: any;
@@ -86,12 +82,15 @@ export class OktaDirectoryService extends BaseDirectoryService implements Direct
             if (oktaFilter != null) {
                 deactOktaFilter = '(' + oktaFilter + ') and ' + deactOktaFilter;
             }
-            deactUsersPromise = this.client.listUsers({ filter: deactOktaFilter }).each((user: any) => {
-                const entry = this.buildUser(user);
-                if (entry != null && !this.filterOutResult(setFilter, entry.email)) {
-                    entries.push(entry);
-                }
-            });
+            deactUsersPromise = this.apiGetMany('users?filter=' + this.encodeUrlParameter(deactOktaFilter))
+                .then((users: any[]) => {
+                    for (const user of users) {
+                        const entry = this.buildUser(user);
+                        if (entry != null && !this.filterOutResult(setFilter, entry.email)) {
+                            entries.push(entry);
+                        }
+                    }
+                });
         } else {
             deactUsersPromise = Promise.resolve();
         }
@@ -116,10 +115,14 @@ export class OktaDirectoryService extends BaseDirectoryService implements Direct
         const oktaFilter = this.buildOktaFilter(this.syncConfig.groupFilter, force, lastSync);
 
         this.logService.info('Querying groups.');
-        await this.client.listGroups({ filter: oktaFilter }).each(async (group: any) => {
-            const entry = await this.buildGroup(group);
-            if (entry != null && !this.filterOutResult(setFilter, entry.name)) {
-                entries.push(entry);
+        await this.apiGetMany('groups?filter=' + this.encodeUrlParameter(oktaFilter)).then(async (groups: any[]) => {
+            for (const group of groups) {
+                const entry = await this.buildGroup(group);
+                if (entry != null && !this.filterOutResult(setFilter, entry.name)) {
+                    entries.push(entry);
+                }
+                // throttle some to avoid rate limiting
+                await new Promise((resolve) => setTimeout(resolve, 500));
             }
         });
         return entries;
@@ -131,8 +134,10 @@ export class OktaDirectoryService extends BaseDirectoryService implements Direct
         entry.referenceId = group.id;
         entry.name = group.profile.name;
 
-        await this.client.listGroupUsers(group.id).each((user: any) => {
-            entry.userMemberExternalIds.add(user.id);
+        await this.apiGetMany('groups/' + group.id + '/users').then((users: any[]) => {
+            for (const user of users) {
+                entry.userMemberExternalIds.add(user.id);
+            }
         });
 
         return entry;
@@ -152,4 +157,88 @@ export class OktaDirectoryService extends BaseDirectoryService implements Direct
 
         return '(' + baseFilter + ') and ' + updatedFilter;
     }
+
+    private encodeUrlParameter(filter: string): string {
+        return filter == null ? '' : encodeURIComponent(filter);
+    }
+
+    private async apiGetCall(url: string): Promise<[any, Map<string, string | string[]>]> {
+        const u = new URL(url);
+        return new Promise((resolve) => {
+            https.get({
+                hostname: u.hostname,
+                path: u.pathname + u.search,
+                port: 443,
+                headers: {
+                    Authorization: 'SSWS ' + this.dirConfig.token,
+                    Accept: 'application/json',
+                },
+            }, (res) => {
+                let body = '';
+
+                res.on('data', (chunk) => {
+                    body += chunk;
+                });
+
+                res.on('end', () => {
+                    if (res.statusCode !== 200) {
+                        resolve(null);
+                        return;
+                    }
+
+                    const responseJson = JSON.parse(body);
+                    if (res.headers != null) {
+                        const headersMap = new Map<string, string | string[]>();
+                        for (const key in res.headers) {
+                            if (res.headers.hasOwnProperty(key)) {
+                                const val = res.headers[key];
+                                headersMap.set(key.toLowerCase(), val);
+                            }
+                        }
+                        resolve([responseJson, headersMap]);
+                        return;
+                    }
+                    resolve([responseJson, null]);
+                });
+
+                res.on('error', () => {
+                    resolve(null);
+                });
+            });
+        });
+    }
+
+    private async apiGetMany(endpoint: string, currentData: any[] = []): Promise<any[]> {
+        const url = endpoint.indexOf('https://') === 0 ? endpoint : `${this.dirConfig.orgUrl}/api/v1/${endpoint}`;
+        const response = await this.apiGetCall(url);
+        if (response == null || response[0] == null || !Array.isArray(response[0])) {
+            throw new Error('API call failed.');
+        }
+        if (response[0].length === 0) {
+            return currentData;
+        }
+        currentData = currentData.concat(response[0]);
+        if (response[1] == null) {
+            return currentData;
+        }
+        const linkHeader = response[1].get('link');
+        if (linkHeader == null || Array.isArray(linkHeader)) {
+            return currentData;
+        }
+        let nextLink: string = null;
+        const linkHeaderParts = linkHeader.split(',');
+        for (const part of linkHeaderParts) {
+            if (part.indexOf('; rel="next"') > -1) {
+                const subParts = part.split(';');
+                if (subParts.length > 0 && subParts[0].indexOf('https://') > -1) {
+                    nextLink = subParts[0].replace('>', '').replace('<', '').trim();
+                    break;
+                }
+            }
+        }
+        if (nextLink == null) {
+            return currentData;
+        }
+        return this.apiGetMany(nextLink, currentData);
+    }
 }