From 19a02b3732ba042e235b2ee5d0ac8a857347e9c2 Mon Sep 17 00:00:00 2001 From: Kyle Spearrin Date: Wed, 17 May 2017 00:01:03 -0400 Subject: [PATCH] handle group changes more appropriately --- src/Console/Program.cs | 2 +- src/Core/Models/Entry.cs | 4 +- src/Core/Models/ImportRequest.cs | 2 +- src/Core/Services/AzureDirectoryService.cs | 60 +++++--- src/Core/Services/LdapDirectoryService.cs | 163 ++++++++++++++------- src/Core/Utilities/Sync.cs | 26 ++-- 6 files changed, 162 insertions(+), 95 deletions(-) diff --git a/src/Console/Program.cs b/src/Console/Program.cs index 2042c099..8996bba1 100644 --- a/src/Console/Program.cs +++ b/src/Console/Program.cs @@ -702,7 +702,7 @@ namespace Bit.Console foreach(var group in result.Groups) { Con.WriteLine(" {0} - {1}", group.Name, group.ExternalId); - foreach(var user in group.Users) + foreach(var user in group.UserMemberExternalIds) { Con.WriteLine(" {0}", user); } diff --git a/src/Core/Models/Entry.cs b/src/Core/Models/Entry.cs index 2f70b0a9..b62d0663 100644 --- a/src/Core/Models/Entry.cs +++ b/src/Core/Models/Entry.cs @@ -17,8 +17,8 @@ namespace Bit.Core.Models public class GroupEntry : Entry { public string Name { get; set; } - public HashSet Members { get; set; } = new HashSet(); - public HashSet Users { get; set; } = new HashSet(); + public HashSet UserMemberExternalIds { get; set; } = new HashSet(); + public HashSet GroupMemberReferenceIds { get; set; } = new HashSet(); } public class UserEntry : Entry diff --git a/src/Core/Models/ImportRequest.cs b/src/Core/Models/ImportRequest.cs index 13e48bea..ca6f93a3 100644 --- a/src/Core/Models/ImportRequest.cs +++ b/src/Core/Models/ImportRequest.cs @@ -20,7 +20,7 @@ namespace Bit.Core.Models { Name = entry.Name; ExternalId = entry.ExternalId; - Users = entry.Users; + Users = entry.UserMemberExternalIds; } public string Name { get; set; } diff --git a/src/Core/Services/AzureDirectoryService.cs b/src/Core/Services/AzureDirectoryService.cs index f639d142..9e4a891a 100644 --- a/src/Core/Services/AzureDirectoryService.cs +++ b/src/Core/Services/AzureDirectoryService.cs @@ -88,29 +88,47 @@ namespace Bit.Core.Services } var entries = new List(); - - var groupRequest = _graphClient.Groups.Delta(); - IGroupDeltaCollectionPage groups = null; - + var changedGroupIds = new List(); if(SettingsService.Instance.GroupDeltaToken != null) { try { - var delataRequest = groupRequest.Request(); + var delataRequest = _graphClient.Groups.Delta().Request(); delataRequest.QueryOptions.Add(new QueryOption("$deltatoken", SettingsService.Instance.GroupDeltaToken)); - groups = await delataRequest.GetAsync(); - } - catch - { - groups = null; + var groupsDelta = await delataRequest.GetAsync(); + + while(true) + { + changedGroupIds.AddRange(groupsDelta.Select(g => g.Id)); + + if(groupsDelta.NextPageRequest == null) + { + object deltaLink; + if(groupsDelta.AdditionalData.TryGetValue("@odata.deltaLink", out deltaLink)) + { + var deltaUriQuery = new Uri(deltaLink.ToString()).ParseQueryString(); + if(deltaUriQuery["$deltatoken"] != null) + { + SettingsService.Instance.GroupDeltaToken = deltaUriQuery["$deltatoken"]; + } + } + break; + } + else + { + groupsDelta = await groupsDelta.NextPageRequest.GetAsync(); + } + } } + catch { } } - if(groups == null) + if(!changedGroupIds.Any()) { - groups = await groupRequest.Request().GetAsync(); + return entries; } + var groups = await _graphClient.Groups.Request().GetAsync(); while(true) { foreach(var group in groups) @@ -125,7 +143,14 @@ namespace Bit.Core.Services var members = await _graphClient.Groups[group.Id].Members.Request().Select("id").GetAsync(); foreach(var member in members) { - entry.Members.Add(member.Id); + if(member is User) + { + entry.UserMemberExternalIds.Add(member.Id); + } + else if(member is Group) + { + entry.GroupMemberReferenceIds.Add(member.Id); + } } entries.Add(entry); @@ -133,15 +158,6 @@ namespace Bit.Core.Services if(groups.NextPageRequest == null) { - object deltaLink; - if(groups.AdditionalData.TryGetValue("@odata.deltaLink", out deltaLink)) - { - var deltaUriQuery = new Uri(deltaLink.ToString()).ParseQueryString(); - if(deltaUriQuery["$deltatoken"] != null) - { - SettingsService.Instance.GroupDeltaToken = deltaUriQuery["$deltatoken"]; - } - } break; } else diff --git a/src/Core/Services/LdapDirectoryService.cs b/src/Core/Services/LdapDirectoryService.cs index 0d30d89d..03f62302 100644 --- a/src/Core/Services/LdapDirectoryService.cs +++ b/src/Core/Services/LdapDirectoryService.cs @@ -83,11 +83,14 @@ namespace Bit.Core.Services } var entry = SettingsService.Instance.Server.Ldap.GetDirectoryEntry(); - var filter = string.IsNullOrWhiteSpace(SettingsService.Instance.Sync.GroupFilter) ? null : + + string originalFilter; + var filter = originalFilter = string.IsNullOrWhiteSpace(SettingsService.Instance.Sync.GroupFilter) ? null : SettingsService.Instance.Sync.GroupFilter; - if(!force && !string.IsNullOrWhiteSpace(SettingsService.Instance.Sync.RevisionDateAttribute) && - SettingsService.Instance.LastGroupSyncDate.HasValue) + var searchSinceRevision = !force && !string.IsNullOrWhiteSpace(SettingsService.Instance.Sync.RevisionDateAttribute) && + SettingsService.Instance.LastGroupSyncDate.HasValue; + if(searchSinceRevision) { filter = string.Format("(&{0}({1}>{2}))", filter != null ? string.Format("({0})", filter) : string.Empty, @@ -98,68 +101,124 @@ namespace Bit.Core.Services var searcher = new DirectorySearcher(entry, filter); var result = searcher.FindAll(); + var initialSearchGroupIds = new List(); + foreach(SearchResult item in result) + { + initialSearchGroupIds.Add(new Uri(item.Path).Segments?.LastOrDefault()); + } + + if(searchSinceRevision && !initialSearchGroupIds.Any()) + { + return Task.FromResult(new List()); + } + else if(searchSinceRevision) + { + searcher = new DirectorySearcher(entry, originalFilter); + result = searcher.FindAll(); + } + + var userFilter = string.IsNullOrWhiteSpace(SettingsService.Instance.Sync.UserFilter) ? null : + SettingsService.Instance.Sync.UserFilter; + var userSearcher = new DirectorySearcher(entry, userFilter); + var userResult = userSearcher.FindAll(); + + var userIdsDict = MakeIdIndex(userResult); + var groups = new List(); foreach(SearchResult item in result) { - var group = new GroupEntry - { - ReferenceId = new Uri(item.Path).Segments?.LastOrDefault() - }; - - if(group.ReferenceId == null) + var group = BuildGroup(item, userIdsDict); + if(group == null) { continue; } - // External Id - if(item.Properties.Contains("objectGUID") && item.Properties["objectGUID"].Count > 0) - { - group.ExternalId = item.Properties["objectGUID"][0].ToString(); - } - else - { - group.ExternalId = group.ReferenceId; - } - - // Name - if(item.Properties.Contains(SettingsService.Instance.Sync.GroupNameAttribute) && - item.Properties[SettingsService.Instance.Sync.GroupNameAttribute].Count > 0) - { - group.Name = item.Properties[SettingsService.Instance.Sync.GroupNameAttribute][0].ToString(); - } - else if(item.Properties.Contains("cn") && item.Properties["cn"].Count > 0) - { - group.Name = item.Properties["cn"][0].ToString(); - } - else - { - continue; - } - - // Dates - group.CreationDate = item.Properties.ParseDateTime(SettingsService.Instance.Sync.CreationDateAttribute); - group.RevisionDate = item.Properties.ParseDateTime(SettingsService.Instance.Sync.RevisionDateAttribute); - - // Members - if(item.Properties.Contains(SettingsService.Instance.Sync.MemberAttribute) && - item.Properties[SettingsService.Instance.Sync.MemberAttribute].Count > 0) - { - foreach(var member in item.Properties[SettingsService.Instance.Sync.MemberAttribute]) - { - var memberDn = member.ToString(); - if(!group.Members.Contains(memberDn)) - { - group.Members.Add(memberDn); - } - } - } - groups.Add(group); } return Task.FromResult(groups); } + private static Dictionary MakeIdIndex(SearchResultCollection result) + { + var dict = new Dictionary(); + foreach(SearchResult item in result) + { + var referenceId = new Uri(item.Path).Segments?.LastOrDefault(); + var externalId = referenceId; + + if(item.Properties.Contains("objectGUID") && item.Properties["objectGUID"].Count > 0) + { + externalId = item.Properties["objectGUID"][0].ToString(); + } + + dict.Add(referenceId, externalId); + } + return dict; + } + + private static GroupEntry BuildGroup(SearchResult item, Dictionary userIndex) + { + var group = new GroupEntry + { + ReferenceId = new Uri(item.Path).Segments?.LastOrDefault() + }; + + if(group.ReferenceId == null) + { + return null; + } + + // External Id + if(item.Properties.Contains("objectGUID") && item.Properties["objectGUID"].Count > 0) + { + group.ExternalId = item.Properties["objectGUID"][0].ToString(); + } + else + { + group.ExternalId = group.ReferenceId; + } + + // Name + if(item.Properties.Contains(SettingsService.Instance.Sync.GroupNameAttribute) && + item.Properties[SettingsService.Instance.Sync.GroupNameAttribute].Count > 0) + { + group.Name = item.Properties[SettingsService.Instance.Sync.GroupNameAttribute][0].ToString(); + } + else if(item.Properties.Contains("cn") && item.Properties["cn"].Count > 0) + { + group.Name = item.Properties["cn"][0].ToString(); + } + else + { + return null; + } + + // Dates + group.CreationDate = item.Properties.ParseDateTime(SettingsService.Instance.Sync.CreationDateAttribute); + group.RevisionDate = item.Properties.ParseDateTime(SettingsService.Instance.Sync.RevisionDateAttribute); + + // Members + if(item.Properties.Contains(SettingsService.Instance.Sync.MemberAttribute) && + item.Properties[SettingsService.Instance.Sync.MemberAttribute].Count > 0) + { + foreach(var member in item.Properties[SettingsService.Instance.Sync.MemberAttribute]) + { + var memberDn = member.ToString(); + if(userIndex.ContainsKey(memberDn) && !group.UserMemberExternalIds.Contains(userIndex[memberDn])) + { + group.UserMemberExternalIds.Add(userIndex[memberDn]); + } + else if(!group.GroupMemberReferenceIds.Contains(memberDn)) + { + group.GroupMemberReferenceIds.Add(memberDn); + } + } + } + + return group; + } + private static Task> GetUsersAsync(bool force = false) { if(!SettingsService.Instance.Sync.SyncUsers) diff --git a/src/Core/Utilities/Sync.cs b/src/Core/Utilities/Sync.cs index 8b0d3816..49ab63f9 100644 --- a/src/Core/Utilities/Sync.cs +++ b/src/Core/Utilities/Sync.cs @@ -21,7 +21,7 @@ namespace Bit.Core.Utilities var groups = entriesResult.Item1; var users = entriesResult.Item2; - FlattenUsersToGroups(groups, null, groups, users); + FlattenUsersToGroups(groups, null, groups); if(!sendToServer || (groups.Count == 0 && users.Count == 0)) { @@ -92,29 +92,21 @@ namespace Bit.Core.Utilities } } - private static void FlattenUsersToGroups(List currentGroups, List currentGroupsUsers, - List allGroups, List allUsers) + private static void FlattenUsersToGroups(List currentGroups, List currentGroupsUsers, + List allGroups) { foreach(var group in currentGroups) { - var groupsInThisGroup = allGroups.Where(g => group.Members.Contains(g.ReferenceId)).ToList(); - var usersInThisGroup = allUsers.Where(u => group.Members.Contains(u.ReferenceId)).ToList(); - - foreach(var user in usersInThisGroup) - { - if(!group.Users.Contains(user.ExternalId)) - { - group.Users.Add(user.ExternalId); - } - } + var groupsInThisGroup = allGroups.Where(g => group.GroupMemberReferenceIds.Contains(g.ReferenceId)).ToList(); + var usersInThisGroup = group.UserMemberExternalIds.ToList(); if(currentGroupsUsers != null) { - foreach(var user in currentGroupsUsers) + foreach(var id in currentGroupsUsers) { - if(!group.Users.Contains(user.ExternalId)) + if(!group.UserMemberExternalIds.Contains(id)) { - group.Users.Add(user.ExternalId); + group.UserMemberExternalIds.Add(id); } } @@ -122,7 +114,7 @@ namespace Bit.Core.Utilities } // Recurse it - FlattenUsersToGroups(groupsInThisGroup, usersInThisGroup, allGroups, allUsers); + FlattenUsersToGroups(groupsInThisGroup, usersInThisGroup, allGroups); } }