From 42b0113faea68905ce078a9b26b3624f879071a1 Mon Sep 17 00:00:00 2001 From: Bryan Roe Date: Thu, 26 Mar 2020 19:34:54 -0700 Subject: [PATCH] Added support for minimum db.compact() threshold --- meshcore/agentcore.c | 1 + meshcore/agentcore.h | 1 + microstack/ILibSimpleDataStore.c | 40 +++++++++++++++++++++++++++----- microstack/ILibSimpleDataStore.h | 2 ++ 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/meshcore/agentcore.c b/meshcore/agentcore.c index fef8a7d..0bb555d 100644 --- a/meshcore/agentcore.c +++ b/meshcore/agentcore.c @@ -3468,6 +3468,7 @@ void MeshServer_Connect(MeshAgentHostContainer *agent) util_random(sizeof(int), (char*)&timeout); gRemoteMouseRenderDefault = ILibSimpleDataStore_Get(agent->masterDb, "remoteMouseRender", NULL, 0); + ILibSimpleDataStore_ConfigCompact(agent->masterDb, ILibSimpleDataStore_GetInt(agent->masterDb, "compactDirtyMinimum", 0)); agent->disableUpdate = ILibSimpleDataStore_Get(agent->masterDb, "disableUpdate", NULL, 0); agent->forceUpdate = ILibSimpleDataStore_Get(agent->masterDb, "forceUpdate", NULL, 0); agent->logUpdate = ILibSimpleDataStore_Get(agent->masterDb, "logUpdate", NULL, 0); diff --git a/meshcore/agentcore.h b/meshcore/agentcore.h index 4aae25a..293fe90 100644 --- a/meshcore/agentcore.h +++ b/meshcore/agentcore.h @@ -249,6 +249,7 @@ char* MeshAgent_MakeAbsolutePath(char *basePath, char *localPath); * AgentCapabilities: Integer Mask, specifying supported Agent Capabilities agentName If set, this will be sent to the server instead of the hostname +compactDirtyMinimum Minimum dirty bytes threshold for db.compact() operation controlChannelDebug: If set, will log/display controlChannel messages (Except for JSON messages) controlChannelIdleTimeout: Integer value specifying the idle timeout in seconds, to send Ping/Pong to server, to keep connection alive coreDumpEnabled: If set, a dump file will be written when the agent crashes diff --git a/microstack/ILibSimpleDataStore.c b/microstack/ILibSimpleDataStore.c index e780c4e..1aa7c1f 100644 --- a/microstack/ILibSimpleDataStore.c +++ b/microstack/ILibSimpleDataStore.c @@ -48,6 +48,8 @@ typedef struct ILibSimpleDataStore_Root ILibHashtable keyTable; // keys --> ILibSimpleDataStore_TableEntry ILibHashtable cacheTable; uint64_t fileSize; + uint64_t dirtySize; + uint64_t minimumDirtySize; int error; } ILibSimpleDataStore_Root; @@ -388,7 +390,17 @@ void ILibSimpleDataStore_RebuildKeyTable(ILibSimpleDataStore_Root *root) if (node->valueLength > 0) { // If the value is not empty, we need to create/overwrite this value in memory - if (entry == NULL) { ++count; entry = (ILibSimpleDataStore_TableEntry*)ILibMemory_Allocate(sizeof(ILibSimpleDataStore_TableEntry), 0, NULL, NULL); } + if (entry == NULL) + { + // Create new entry in table + ++count; + entry = (ILibSimpleDataStore_TableEntry*)ILibMemory_Allocate(sizeof(ILibSimpleDataStore_TableEntry), 0, NULL, NULL); + } + else + { + // Entry already exists in table + root->dirtySize += entry->valueLength; + } memcpy_s(entry->valueHash, sizeof(entry->valueHash), node->hash, SHA384HASHSIZE); entry->valueLength = node->valueLength; entry->valueOffset = ILibSimpleDataStore_RecordHeader_ValueOffset(node); @@ -397,6 +409,7 @@ void ILibSimpleDataStore_RebuildKeyTable(ILibSimpleDataStore_Root *root) else if (entry != NULL) { // If value is empty, remove the in-memory entry. + root->dirtySize += entry->valueLength; --count; ILibHashtable_Remove(root->keyTable, NULL, node->key, node->keyLen); free(entry); @@ -599,10 +612,14 @@ __EXPORT_TYPE int ILibSimpleDataStore_PutEx(ILibSimpleDataStore dataStore, char* ILibSimpleDataStore_SHA384(value, valueLen, hash); // Hash the value // Create a new record for the key and value - if (entry == NULL) { - entry = (ILibSimpleDataStore_TableEntry*)ILibMemory_Allocate(sizeof(ILibSimpleDataStore_TableEntry), 0, NULL, NULL); } - else { + if (entry == NULL) + { + entry = (ILibSimpleDataStore_TableEntry*)ILibMemory_Allocate(sizeof(ILibSimpleDataStore_TableEntry), 0, NULL, NULL); + } + else + { if (memcmp(entry->valueHash, hash, SHA384HASHSIZE) == 0) { return 0; } + root->dirtySize += entry->valueLength; } memcpy_s(entry->valueHash, sizeof(entry->valueHash), hash, SHA384HASHSIZE); @@ -614,6 +631,13 @@ __EXPORT_TYPE int ILibSimpleDataStore_PutEx(ILibSimpleDataStore dataStore, char* return ILibHashtable_Put(root->keyTable, NULL, key, keyLen, entry) == NULL ? 0 : 1; } +__EXPORT_TYPE int ILibSimpleDataStore_GetInt(ILibSimpleDataStore dataStore, char* key, int defaultValue) +{ + int bufLen = ILibSimpleDataStore_Get(dataStore, key, ILibScratchPad, sizeof(ILibScratchPad)); + if (bufLen == 0 || bufLen > sizeof(ILibScratchPad)) { return(defaultValue); } + return(atoi(ILibScratchPad)); +} + // Get a value from the data store given a key __EXPORT_TYPE int ILibSimpleDataStore_GetEx(ILibSimpleDataStore dataStore, char* key, int keyLen, char *buffer, int bufferLen) { @@ -776,7 +800,11 @@ __EXPORT_TYPE void ILibSimpleDataStore_EnumerateKeys(ILibSimpleDataStore dataSto if (handler != NULL) { ILibHashtable_Enumerate(root->keyTable, ILibSimpleDataStore_EnumerateKeysSink, users); } } - +__EXPORT_TYPE void ILibSimpleDataStore_ConfigCompact(ILibSimpleDataStore dataStore, uint64_t minimumDirtySize) +{ + ILibSimpleDataStore_Root *root = (ILibSimpleDataStore_Root*)dataStore; + root->minimumDirtySize = minimumDirtySize; +} // Compact the data store __EXPORT_TYPE int ILibSimpleDataStore_Compact(ILibSimpleDataStore dataStore) { @@ -786,7 +814,7 @@ __EXPORT_TYPE int ILibSimpleDataStore_Compact(ILibSimpleDataStore dataStore) void* state[2]; int retVal = 0; - if (root == NULL) return 1; // Error + if (root == NULL || root->dirtySize < root->minimumDirtySize) return 1; // Error tmp = ILibString_Cat(root->filePath, -1, ".tmp", -1); // Create the name of the temporary data store // Start by opening a temporary .tmp file. Will be used to write the compacted data store. diff --git a/microstack/ILibSimpleDataStore.h b/microstack/ILibSimpleDataStore.h index f6a847b..1a60fc6 100644 --- a/microstack/ILibSimpleDataStore.h +++ b/microstack/ILibSimpleDataStore.h @@ -54,6 +54,7 @@ __EXPORT_TYPE void ILibSimpleDataStore_Close(ILibSimpleDataStore dataStore); __EXPORT_TYPE void ILibSimpleDataStore_Cached(ILibSimpleDataStore dataStore, char* key, int keyLen, char* value, int valueLen); __EXPORT_TYPE int ILibSimpleDataStore_Cached_GetJSON(ILibSimpleDataStore dataStore, char *buffer, int bufferLen); __EXPORT_TYPE int ILibSimpleDataStore_Cached_GetJSONEx(ILibSimpleDataStore dataStore, char *buffer, int bufferLen); +__EXPORT_TYPE void ILibSimpleDataStore_ConfigCompact(ILibSimpleDataStore dataStore, uint64_t minimumDirtySize); __EXPORT_TYPE int ILibSimpleDataStore_PutEx(ILibSimpleDataStore dataStore, char* key, int keyLen, char* value, int valueLen); #define ILibSimpleDataStore_Put(dataStore, key, value) ILibSimpleDataStore_PutEx(dataStore, key, (int)strnlen_s(key, ILibSimpleDataStore_MaxKeyLength), value, (int)strnlen_s(value, ILibSimpleDataStore_MaxUnspecifiedValueLen)) @@ -61,6 +62,7 @@ __EXPORT_TYPE int ILibSimpleDataStore_PutEx(ILibSimpleDataStore dataStore, char* // Get a value from the datastore of given a key. __EXPORT_TYPE int ILibSimpleDataStore_GetEx(ILibSimpleDataStore dataStore, char* key, int keyLen, char* buffer, int bufferLen); #define ILibSimpleDataStore_Get(dataStore, key, buffer, bufferLen) ILibSimpleDataStore_GetEx(dataStore, key, (int)strnlen_s(key, ILibSimpleDataStore_MaxKeyLength), buffer, bufferLen) +__EXPORT_TYPE int ILibSimpleDataStore_GetInt(ILibSimpleDataStore dataStore, char* key, int defaultValue); // Get the SHA256 hash value from the datastore for a given a key. __EXPORT_TYPE char* ILibSimpleDataStore_GetHashEx(ILibSimpleDataStore dataStore, char* key, int keyLen);