From e82255959aa81df3015f83e935e4f7b04ae48e18 Mon Sep 17 00:00:00 2001 From: Bryan Roe Date: Mon, 22 Jun 2020 19:10:54 -0700 Subject: [PATCH] Added support for compressed db entries --- microscript/ILibDuktape_CompressedStream.c | 171 +++++++++++++++++++++ microstack/ILibSimpleDataStore.c | 132 +++++++++++++++- microstack/ILibSimpleDataStore.h | 4 +- 3 files changed, 301 insertions(+), 6 deletions(-) diff --git a/microscript/ILibDuktape_CompressedStream.c b/microscript/ILibDuktape_CompressedStream.c index b2b9741..0b8495e 100644 --- a/microscript/ILibDuktape_CompressedStream.c +++ b/microscript/ILibDuktape_CompressedStream.c @@ -326,15 +326,186 @@ duk_ret_t ILibDuktape_CompressedStream_decompressor(duk_context *ctx) duk_events_newListener2(ctx, -1, "data", ILibDuktape_CompressedStream_resume_newListener); return(1); } +duk_ret_t ILibDuktape_CompressedStream_deflate_dataSink(duk_context *ctx) +{ + duk_push_this(ctx); // [stream] + duk_push_array(ctx); // [stream][array] + if (duk_has_prop_string(ctx, -2, "_buf")) + { + duk_get_prop_string(ctx, -2, "_buf"); // [stream][array][buffer] + duk_array_push(ctx, -2); // [stream][array] + } + duk_dup(ctx, 0); // [stream][array][buffer] + duk_array_push(ctx, -2); // [stream][array] + duk_buffer_concat(ctx); // [stream][buffer] + duk_put_prop_string(ctx, -2, "_buf"); // [stream] + return(0); +} + +duk_ret_t ILibDuktape_CompressedStream_deflate(duk_context *ctx) +{ + duk_eval_string(ctx, "require('compressed-stream').createCompressor({WBIT: -15});");// [decompressor] + ILibDuktape_EventEmitter_AddOnEx(ctx, -1, "data", ILibDuktape_CompressedStream_deflate_dataSink); + duk_prepare_method_call(ctx, -1, "end"); // [decompressor][end][this] + duk_dup(ctx, 0); // [decompressor][end][this][buffer] + duk_pcall_method(ctx, 1); // [decompressor][val] + duk_get_prop_string(ctx, -2, "_buf"); // [decompressor][val][buffer] + return(1); +} +duk_ret_t ILibDuktape_CompressedStream_inflate(duk_context *ctx) +{ + duk_eval_string(ctx, "require('compressed-stream').createDecompressor({WBIT: -15});");// [decompressor] + ILibDuktape_EventEmitter_AddOnEx(ctx, -1, "data", ILibDuktape_CompressedStream_deflate_dataSink); + duk_prepare_method_call(ctx, -1, "end"); // [decompressor][end][this] + duk_dup(ctx, 0); // [decompressor][end][this][buffer] + duk_pcall_method(ctx, 1); // [decompressor][val] + duk_get_prop_string(ctx, -2, "_buf"); // [decompressor][val][buffer] + return(1); +} void ILibDuktape_CompressedStream_PUSH(duk_context *ctx, void *chain) { duk_push_object(ctx); // [compressed-stream] ILibDuktape_WriteID(ctx, "compressedStream"); ILibDuktape_CreateInstanceMethod(ctx, "createCompressor", ILibDuktape_CompressedStream_compressor, DUK_VARARGS); ILibDuktape_CreateInstanceMethod(ctx, "createDecompressor", ILibDuktape_CompressedStream_decompressor, DUK_VARARGS); + ILibDuktape_CreateInstanceMethod(ctx, "deflate", ILibDuktape_CompressedStream_deflate, DUK_VARARGS); + ILibDuktape_CreateInstanceMethod(ctx, "inflate", ILibDuktape_CompressedStream_inflate, DUK_VARARGS); } void ILibDuktape_CompressedStream_init(duk_context * ctx) { ILibDuktape_ModSearch_AddHandler(ctx, "compressed-stream", ILibDuktape_CompressedStream_PUSH); } + +int ILibDeflate(char *buffer, size_t bufferLen, char *compressed, size_t *compressedLen, uint32_t *crc) +{ + int ret = 0; + z_stream Z; + char tmp[16384]; + size_t avail; + size_t len = 0; + uint32_t rescrc = 0; + + if (compressed != NULL) { len = *compressedLen; *compressedLen = 0; } + + memset(&Z, 0, sizeof(Z)); + Z.zalloc = Z_NULL; + Z.zfree = Z_NULL; + Z.opaque = Z_NULL; + if (deflateInit2(&Z, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) != Z_OK) { return(1); } + + Z.avail_in = (uInt)bufferLen; + Z.next_in = (Bytef*)buffer; + + do + { + Z.avail_out = sizeof(tmp); + Z.next_out = (Bytef*)tmp; + ignore_result(deflate(&(Z), Z_NO_FLUSH)); + avail = sizeof(tmp) - Z.avail_out; + if (avail > 0) + { + rescrc = crc32(rescrc, (unsigned char*)tmp, (unsigned int)avail); + if (compressed != NULL) + { + if ((len - *compressedLen) < avail) { ret = 1; break; } + memcpy_s(compressed + *compressedLen, len - *compressedLen, (char*)tmp, avail); + } + *compressedLen += avail; + } + } while (Z.avail_out == 0); + + Z.avail_in = 0; + Z.next_in = (Bytef*)ILibScratchPad; + + if (ret == 0) + { + do + { + Z.avail_out = sizeof(tmp); + Z.next_out = (Bytef*)tmp; + ignore_result(deflate(&Z, Z_FINISH)); + avail = sizeof(tmp) - Z.avail_out; + if (avail > 0) + { + rescrc = crc32(rescrc, (unsigned char*)tmp, (unsigned int)avail); + if (compressed != NULL) + { + if ((len - *compressedLen) < avail) { ret = 1; break; } + memcpy_s(compressed + *compressedLen, len - *compressedLen, (char*)tmp, avail); + } + *compressedLen += avail; + } + } while (Z.avail_out == 0); + } + ignore_result(deflateEnd(&Z)); + if (crc != NULL) { *crc = rescrc; } + return(ret); +} + +int ILibInflate(char *buffer, size_t bufferLen, char *decompressed, size_t *decompressedLen, uint32_t crc) +{ + int ret = 0; + z_stream Z; + char tmp[16384]; + size_t avail; + size_t len = 0; + uint32_t rescrc = 0; + + if (decompressed != NULL) { len = *decompressedLen; *decompressedLen = 0; } + + memset(&Z, 0, sizeof(Z)); + Z.zalloc = Z_NULL; + Z.zfree = Z_NULL; + Z.opaque = Z_NULL; + if (inflateInit2(&Z, -MAX_WBITS) != Z_OK) { return(1); } + + Z.avail_in = (uInt)bufferLen; + Z.next_in = (Bytef*)buffer; + + do + { + Z.avail_out = sizeof(tmp); + Z.next_out = (Bytef*)tmp; + ignore_result(inflate(&Z, Z_NO_FLUSH)); + avail = sizeof(tmp) - Z.avail_out; + if (avail > 0) + { + rescrc = crc32(rescrc, (unsigned char*)tmp, (unsigned int)avail); + if (decompressed != NULL) + { + if (avail > (len - *decompressedLen)) { ret = 1; break; } + memcpy_s(decompressed + *decompressedLen, len - *decompressedLen, tmp, avail); + } + *decompressedLen += avail; + } + } while (Z.avail_out == 0); + + if (ret == 0) + { + Z.avail_in = 0; + Z.next_in = (Bytef*)ILibScratchPad; + + do + { + Z.avail_out = sizeof(tmp); + Z.next_out = (Bytef*)tmp; + ignore_result(inflate(&Z, Z_FINISH)); + avail = sizeof(tmp) - Z.avail_out; + if (avail > 0) + { + rescrc = crc32(rescrc, (unsigned char*)tmp, (unsigned int)avail); + if (decompressed != NULL) + { + if (avail > (len - *decompressedLen)) { ret = 1; break; } + memcpy_s(decompressed + *decompressedLen, len - *decompressedLen, tmp, avail); + } + *decompressedLen += avail; + } + } while (Z.avail_out == 0); + } + ignore_result(inflateEnd(&Z)); + + if (crc != 0 && crc != rescrc) { ret = 2; } + return(ret); +} \ No newline at end of file diff --git a/microstack/ILibSimpleDataStore.c b/microstack/ILibSimpleDataStore.c index 8f8040b..48c198f 100644 --- a/microstack/ILibSimpleDataStore.c +++ b/microstack/ILibSimpleDataStore.c @@ -112,7 +112,9 @@ typedef struct ILibSimpleDataStore_CacheEntry const int ILibMemory_SimpleDataStore_CONTAINERSIZE = sizeof(ILibSimpleDataStore_Root); void ILibSimpleDataStore_RebuildKeyTable(ILibSimpleDataStore_Root *root); - +extern int ILibInflate(char *buffer, size_t bufferLen, char *decompressed, size_t *decompressedLen, uint32_t crc); +extern int ILibDeflate(char *buffer, size_t bufferLen, char *compressed, size_t *compressedLen, uint32_t *crc); +extern uint32_t crc32c(uint32_t crci, const unsigned char *buf, uint32_t len); // Perform a SHA384 hash of some data void ILibSimpleDataStore_SHA384(char *data, int datalen, char* result) { util_sha384(data, datalen, result); } @@ -353,6 +355,16 @@ ILibSimpleDataStore_RecordHeader_NG* ILibSimpleDataStore_ReadNextRecord(ILibSimp { // Check the hash if (memcmp(node->hash, result, SHA384HASHSIZE) == 0) { return node; } // Data is correct + + // Before we assume this is a bad hash check, we need to verify it's not a compressed node + if (node->keyLen > sizeof(uint32_t)) + { + if (crc32c(0, node->key, node->keyLen - sizeof(uint32_t)) == ((uint32_t*)(node->key + node->keyLen - sizeof(uint32_t)))[0]) + { + return(node); + } + } + return NULL; // Data is corrupt } return node; @@ -608,7 +620,7 @@ __EXPORT_TYPE void ILibSimpleDataStore_Close(ILibSimpleDataStore dataStore) } // Store a key/value pair in the data store -__EXPORT_TYPE int ILibSimpleDataStore_PutEx(ILibSimpleDataStore dataStore, char* key, int keyLen, char* value, int valueLen) +__EXPORT_TYPE int ILibSimpleDataStore_PutEx2(ILibSimpleDataStore dataStore, char* key, int keyLen, char* value, int valueLen, char *vhash) { int ret; char hash[SHA384HASHSIZE]; @@ -623,8 +635,18 @@ __EXPORT_TYPE int ILibSimpleDataStore_PutEx(ILibSimpleDataStore dataStore, char* } if (keyLen > 1 && key[keyLen - 1] == 0) { keyLen -= 1; } + if (vhash != NULL) + { + char *tmpkey = (char*)ILibMemory_SmartAllocate(keyLen + sizeof(int)); + memcpy_s(tmpkey, ILibMemory_Size(tmpkey), key, keyLen); + ((uint32_t*)(tmpkey + keyLen))[0] = crc32c(0, tmpkey, keyLen); + key = tmpkey; + keyLen = (int)ILibMemory_Size(key); + memcpy_s(hash, sizeof(hash), vhash, SHA384HASHSIZE); + } + entry = (ILibSimpleDataStore_TableEntry*)ILibHashtable_Get(root->keyTable, NULL, key, keyLen); - ILibSimpleDataStore_SHA384(value, valueLen, hash); // Hash the value + if (vhash == NULL) { ILibSimpleDataStore_SHA384(value, valueLen, hash); } // Hash the value // Create a new record for the key and value if (entry == NULL) @@ -651,6 +673,28 @@ __EXPORT_TYPE int ILibSimpleDataStore_PutEx(ILibSimpleDataStore dataStore, char* return(ret); } +int ILibSimpleDataStore_PutCompressed(ILibSimpleDataStore dataStore, char* key, int keyLen, char* value, int valueLen) +{ + int ret = 1; + char hash[SHA384HASHSIZE]; + char *tmp = NULL; + size_t tmpLen = 0; + uint32_t crc = 0; + if (ILibDeflate(value, valueLen, NULL, &tmpLen, NULL) == 0) + { + tmp = (char*)ILibMemory_SmartAllocate(tmpLen); + if (ILibDeflate(value, valueLen, tmp, &tmpLen, NULL) == 0) + { + ILibSimpleDataStore_SHA384(value, valueLen, hash); // Hash the Uncompressed Data + ILibSimpleDataStore_PutEx2(dataStore, key, keyLen, tmp, (int)tmpLen, hash); + ret = 0; + } + ILibMemory_Free(tmp); + } + return(ret); +} + + __EXPORT_TYPE int ILibSimpleDataStore_GetInt(ILibSimpleDataStore dataStore, char* key, int defaultValue) { int bufLen = ILibSimpleDataStore_Get(dataStore, key, ILibScratchPad, sizeof(ILibScratchPad)); @@ -661,6 +705,7 @@ __EXPORT_TYPE int ILibSimpleDataStore_GetInt(ILibSimpleDataStore dataStore, char // Get a value from the data store given a key __EXPORT_TYPE int ILibSimpleDataStore_GetEx(ILibSimpleDataStore dataStore, char* key, int keyLen, char *buffer, int bufferLen) { + int isCompressed = 0; char hash[SHA384HASHSIZE]; ILibSimpleDataStore_Root *root = (ILibSimpleDataStore_Root*)dataStore; ILibSimpleDataStore_TableEntry *entry; @@ -688,9 +733,19 @@ __EXPORT_TYPE int ILibSimpleDataStore_GetEx(ILibSimpleDataStore dataStore, char* } entry = (ILibSimpleDataStore_TableEntry*)ILibHashtable_Get(root->keyTable, NULL, key, keyLen); + if (entry == NULL) + { + // Before returning an error, check if this is a compressed record + char *tmpkey = (char*)ILibMemory_SmartAllocate(keyLen + sizeof(uint32_t)); + memcpy_s(tmpkey, ILibMemory_Size(tmpkey), key, keyLen); + ((uint32_t*)(tmpkey + keyLen))[0] = crc32c(0, tmpkey, keyLen); + entry = (ILibSimpleDataStore_TableEntry*)ILibHashtable_Get(root->keyTable, NULL, tmpkey, (int)ILibMemory_Size(tmpkey)); + ILibMemory_Free(tmpkey); + if (entry != NULL) { isCompressed = 1; } + } if (entry == NULL) return 0; // If there is no in-memory entry for this key, return zero now. - if ((buffer != NULL) && (bufferLen >= entry->valueLength)) // If the buffer is not null and can hold the value, place the value in the buffer. + if ((buffer != NULL) && (bufferLen >= entry->valueLength) && isCompressed == 0) // If the buffer is not null and can hold the value, place the value in the buffer. { if (ILibSimpleDataStore_SeekPosition(root->dataFile, entry->valueOffset, SEEK_SET) != 0) return 0; // Seek to the position of the value in the data store if (fread(buffer, 1, entry->valueLength, root->dataFile) == 0) return 0; // Read the value into the buffer @@ -698,6 +753,34 @@ __EXPORT_TYPE int ILibSimpleDataStore_GetEx(ILibSimpleDataStore dataStore, char* if (memcmp(hash, entry->valueHash, SHA384HASHSIZE) != 0) return 0; // Check the hash, return 0 if not valid if (bufferLen > entry->valueLength) { buffer[entry->valueLength] = 0; } // Add a zero at the end to be nice, if the buffer can take it. } + else if (isCompressed != 0) + { + // This is a compressed record + char *compressed = ILibMemory_SmartAllocate(entry->valueLength); + size_t tmplen = bufferLen; + if (ILibSimpleDataStore_SeekPosition(root->dataFile, entry->valueOffset, SEEK_SET) != 0) return 0; // Seek to the position of the value in the data store + if (fread(compressed, 1, entry->valueLength, root->dataFile) == 0) return 0; // Read the value into the buffer + if (ILibInflate(compressed, entry->valueLength, buffer, &tmplen, 0) == 0) + { + if (buffer == NULL) { return((int)tmplen); } + + // Before we return, we need to check the HASH of the uncompressed data + char hash[SHA384HASHSIZE]; + ILibSimpleDataStore_SHA384(buffer, (int)tmplen, hash); + if (memcmp(hash, entry->valueHash, SHA384HASHSIZE) == 0) + { + return((int)tmplen); + } + else + { + return(0); + } + } + else + { + return(0); + } + } return entry->valueLength; } @@ -709,6 +792,15 @@ __EXPORT_TYPE char* ILibSimpleDataStore_GetHashEx(ILibSimpleDataStore dataStore, if (root == NULL) return NULL; entry = (ILibSimpleDataStore_TableEntry*)ILibHashtable_Get(root->keyTable, NULL, key, keyLen); + if (entry == NULL) + { + // Before we return an error, let's check if this is a compressed record + char* tmpkey = (char*)ILibMemory_SmartAllocate(keyLen + sizeof(uint32_t)); + memcpy_s(tmpkey, ILibMemory_Size(tmpkey), key, keyLen); + ((uint32_t*)(tmpkey + keyLen))[0] = crc32c(0, key, (uint32_t)keyLen); + entry = (ILibSimpleDataStore_TableEntry*)ILibHashtable_Get(root->keyTable, NULL, tmpkey, (int)ILibMemory_Size(tmpkey)); + ILibMemory_Free(tmpkey); + } if (entry == NULL) return NULL; // If there is no in-memory entry for this key, return zero now. return entry->valueHash; @@ -726,7 +818,28 @@ __EXPORT_TYPE int ILibSimpleDataStore_DeleteEx(ILibSimpleDataStore dataStore, ch if (root == NULL) return 0; entry = (ILibSimpleDataStore_TableEntry*)ILibHashtable_Remove(root->keyTable, NULL, key, keyLen); - if (entry != NULL) { ILibSimpleDataStore_WriteRecord(root->dataFile, key, keyLen, NULL, 0, NULL); free(entry); return 1; } + if (entry == NULL) + { + // Check to see if this is a compressed record, before we return an error + char *tmpkey = (char*)ILibMemory_SmartAllocate(keyLen + sizeof(uint32_t)); + memcpy_s(tmpkey, ILibMemory_Size(tmpkey), key, keyLen); + ((uint32_t*)(tmpkey + keyLen))[0] = crc32c(0, key, keyLen); + entry = (ILibSimpleDataStore_TableEntry*)ILibHashtable_Remove(root->keyTable, NULL, tmpkey, (int)ILibMemory_Size(tmpkey)); + if (entry != NULL) + { + ILibSimpleDataStore_WriteRecord(root->dataFile, tmpkey, (int)ILibMemory_Size(tmpkey), NULL, 0, NULL); + free(entry); + ILibMemory_Free(tmpkey); + return 1; + } + ILibMemory_Free(tmpkey); + } + else + { + ILibSimpleDataStore_WriteRecord(root->dataFile, key, keyLen, NULL, 0, NULL); + free(entry); + return 1; + } return 0; } @@ -808,6 +921,15 @@ void ILibSimpleDataStore_EnumerateKeysSink(ILibHashtable sender, void *Key1, cha UNREFERENCED_PARAMETER(Key2); UNREFERENCED_PARAMETER(Key2Len); + if (Key2Len > sizeof(uint32_t)) + { + // Check if this is a compressed entry + if (crc32c(0, Key2, Key2Len - sizeof(uint32_t)) == ((uint32_t*)(Key2 + Key2Len - sizeof(uint32_t)))[0]) + { + Key2Len -= sizeof(uint32_t); + } + } + handler(dataStore, Key2, Key2Len, userX); // Call the user } diff --git a/microstack/ILibSimpleDataStore.h b/microstack/ILibSimpleDataStore.h index 9706e57..2e8aa10 100644 --- a/microstack/ILibSimpleDataStore.h +++ b/microstack/ILibSimpleDataStore.h @@ -59,8 +59,10 @@ __EXPORT_TYPE int ILibSimpleDataStore_Cached_GetJSONEx(ILibSimpleDataStore dataS __EXPORT_TYPE void ILibSimpleDataStore_ConfigCompact(ILibSimpleDataStore dataStore, uint64_t minimumDirtySize); __EXPORT_TYPE void ILibSimpleDataStore_ConfigSizeLimit(ILibSimpleDataStore dataStore, uint64_t sizeLimit, ILibSimpleDataStore_SizeWarningHandler handler, void *user); -__EXPORT_TYPE int ILibSimpleDataStore_PutEx(ILibSimpleDataStore dataStore, char* key, int keyLen, char* value, int valueLen); +__EXPORT_TYPE int ILibSimpleDataStore_PutEx2(ILibSimpleDataStore dataStore, char* key, int keyLen, char* value, int valueLen, char *hash); #define ILibSimpleDataStore_Put(dataStore, key, value) ILibSimpleDataStore_PutEx(dataStore, key, (int)strnlen_s(key, ILibSimpleDataStore_MaxKeyLength), value, (int)strnlen_s(value, ILibSimpleDataStore_MaxUnspecifiedValueLen)) +#define ILibSimpleDataStore_PutEx(dataStore, key, keyLen, value, valueLen) ILibSimpleDataStore_PutEx2(dataStore, key, keyLen, value, valueLen, NULL) +int ILibSimpleDataStore_PutCompressed(ILibSimpleDataStore dataStore, char* key, int keyLen, char* value, int valueLen); // Get a value from the datastore of given a key. __EXPORT_TYPE int ILibSimpleDataStore_GetEx(ILibSimpleDataStore dataStore, char* key, int keyLen, char* buffer, int bufferLen);