1. Added logging for Windows Cert Store Error cases

2. Added db corruption detection

meshcore/wincrypto.cpp

@@ -181,17 +181,21 @@ wincrypto_object __fastcall wincrypto_open(int newcert, char *rootSubject)
 	ZeroMemory(&exts, sizeof(exts));
 
 	wincrypto_data *ret = (wincrypto_data*)ILibMemory_SmartAllocate(sizeof(wincrypto_data));
-
+	
 	// Open the best CNG possible
-	while (providerName == NULL && wincrypto_CngProviders[r] != NULL) 
+	while (providerName == NULL && wincrypto_CngProviders[r] != NULL)
 	{
 		providerName = wincrypto_CngProviders[r];
 		NCryptOpenStorageProvider(&(ret->hProv), providerName, 0);
 		if (ret->hProv == NULL) providerName = NULL;
 		r++;
 	}
-	if (ret->hProv == NULL) goto error;
-	
+	if (ret->hProv == NULL)
+	{
+		ILIBLOGMESSAGEX("Could not find suitable CngProvider");
+		goto error;
+	}
+
 	// Create cert subject string in format csp understands
 	if (!CertStrToName(X509_ASN_ENCODING, (LPCTSTR)rootSubject, CERT_X500_NAME_STR, NULL, NULL, &subjectEncodedSize, NULL)) goto error;
 	if ((subjectEncoded = (PBYTE)malloc(subjectEncodedSize)) == NULL) ILIBCRITICALEXIT(254);
@@ -199,8 +203,12 @@ wincrypto_object __fastcall wincrypto_open(int newcert, char *rootSubject)
 	sib.cbData = subjectEncodedSize;
 	sib.pbData = subjectEncoded;
 	 
-	ret->hCertStore = CertOpenStore(CERT_STORE_PROV_SYSTEM, X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, ret->hProv, CERT_SYSTEM_STORE_CURRENT_USER | CERT_STORE_OPEN_EXISTING_FLAG, L"MY"); // CERT_STORE_NO_CRYPT_RELEASE_FLAG
-	if (!ret->hCertStore) goto error;
+	ret->hCertStore = CertOpenStore(CERT_STORE_PROV_SYSTEM, X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, NULL, CERT_SYSTEM_STORE_CURRENT_USER | CERT_STORE_OPEN_EXISTING_FLAG, L"MY"); // CERT_STORE_NO_CRYPT_RELEASE_FLAG
+	if (!ret->hCertStore)
+	{
+		ILIBLOGMESSAGEX("Failed to open Windows Cert Store");
+		goto error;
+	}
 
 	// Look for cert and if exists, delete it
 	ret->certCtx = CertFindCertificateInStore(ret->hCertStore, X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, 0, CERT_FIND_SUBJECT_NAME, &sib, NULL );
@@ -208,23 +216,52 @@ wincrypto_object __fastcall wincrypto_open(int newcert, char *rootSubject)
 	// Check if we can get the private key
 	if (ret->certCtx != NULL)
 	{
-		if (!CryptAcquireCertificatePrivateKey(ret->certCtx, CRYPT_ACQUIRE_SILENT_FLAG | CRYPT_ACQUIRE_ALLOW_NCRYPT_KEY_FLAG, NULL, &hKeyNode, &hKeyNodeSpec, &hFreeKeyNode)) { newcert = 1; }
-		if (hKeyNodeSpec != CERT_NCRYPT_KEY_SPEC) { newcert = 1; } // If this private key is not CNG, don't use it.
+		if (!CryptAcquireCertificatePrivateKey(ret->certCtx, CRYPT_ACQUIRE_SILENT_FLAG | CRYPT_ACQUIRE_ALLOW_NCRYPT_KEY_FLAG, NULL, &hKeyNode, &hKeyNodeSpec, &hFreeKeyNode)) 
+		{
+			newcert = 1;
+			ILIBLOGMESSAGEX("CryptAcquireCertificatePrivateKey() Failed");
+		}
+		if (hKeyNodeSpec != CERT_NCRYPT_KEY_SPEC) 
+		{
+			newcert = 1; // If this private key is not CNG, don't use it.
+			ILIBLOGMESSAGEX("hKeyNodeSpec != CERT_NCRYPT_KEY_SPEC");
+		} 
 		if (hFreeKeyNode && hKeyNode != NULL) { if (hKeyNodeSpec == CERT_NCRYPT_KEY_SPEC) NCryptFreeObject(hKeyNode); else CryptReleaseContext(hKeyNode, 0); }
 	}
 
 	// Check if have a certificate already, or need to create a new one
 	if (ret->certCtx != NULL && newcert == 0) goto end;
-	if (ret->certCtx) { status = CertDeleteCertificateFromStore(ret->certCtx); if (!status) goto error; ret->certCtx = NULL; }
+	if (ret->certCtx) 
+	{
+		status = CertDeleteCertificateFromStore(ret->certCtx); 
+		if (!status)
+		{
+			ILIBLOGMESSAGEX("CertDeleteCertificateFromStore() Failed");
+			goto error;
+		}
+		ret->certCtx = NULL;
+	}
 
 	// Generate node RSA key-pair
-    if (FAILED(status = NCryptCreatePersistedKey(ret->hProv, &hKeyNode, BCRYPT_RSA_ALGORITHM, (LPCWSTR)wkeycontainer, 0, NCRYPT_OVERWRITE_KEY_FLAG))) goto error;
+	if (FAILED(status = NCryptCreatePersistedKey(ret->hProv, &hKeyNode, BCRYPT_RSA_ALGORITHM, (LPCWSTR)wkeycontainer, 0, NCRYPT_OVERWRITE_KEY_FLAG)))
+	{
+		ILIBLOGMESSAGEX("NCryptCreatePersistedKey(%s) failed", ILibWideToUTF8(providerName, -1));
+		goto error;
+	}
 	if (FAILED(status = NCryptSetProperty(hKeyNode, NCRYPT_LENGTH_PROPERTY, (PBYTE)&KeyLength, 4, NCRYPT_PERSIST_FLAG | NCRYPT_SILENT_FLAG))) 
 	{
 		KeyLength = 2048; // If 3072 is not supported, go down to 2048.
-		if (FAILED(status = NCryptSetProperty(hKeyNode, NCRYPT_LENGTH_PROPERTY, (PBYTE)&KeyLength, 4, NCRYPT_PERSIST_FLAG | NCRYPT_SILENT_FLAG))) { goto error; }
+		if (FAILED(status = NCryptSetProperty(hKeyNode, NCRYPT_LENGTH_PROPERTY, (PBYTE)&KeyLength, 4, NCRYPT_PERSIST_FLAG | NCRYPT_SILENT_FLAG))) 
+		{
+			ILIBLOGMESSAGEX("NCryptSetProperty(%u) failed", KeyLength);
+			goto error;
+		}
 	}
-	if (FAILED(status = NCryptFinalizeKey(hKeyNode, NCRYPT_SILENT_FLAG))) { goto error; } // Ask for silent create, this will fail if not admin.
+	if (FAILED(status = NCryptFinalizeKey(hKeyNode, NCRYPT_SILENT_FLAG))) // Ask for silent create, this will fail if not admin.
+	{
+		ILIBLOGMESSAGEX("NCryptFinalizeKey() failed");
+		goto error; 
+	} 
 
 	// Create self signed cert 
 	ZeroMemory(&kpi, sizeof(kpi));
@@ -284,15 +321,22 @@ wincrypto_object __fastcall wincrypto_open(int newcert, char *rootSubject)
 	sa.pszObjId = szOID_RSA_SHA384RSA; // Using SHA384
 
 	ret->certCtx = CertCreateSelfSignCertificate(NULL, &sib, 0, &kpi, &sa, &st1, &st2, &exts);
-	if (!ret->certCtx) { goto error; }
+	if (!ret->certCtx) 
+	{
+		ILIBLOGMESSAGEX("CertCreateSelfSignCertificate() failed");
+		goto error; 
+	}
 
 	// Note this is a different context to certCtx, this ctx is the in-store ctx
 	status = CertAddCertificateContextToStore(ret->hCertStore, ret->certCtx, CERT_STORE_ADD_REPLACE_EXISTING, &ret->certCtx);
-	if (!status || ret->certCtx == NULL) goto error;
+	if (!status || ret->certCtx == NULL)
+	{
+		ILIBLOGMESSAGEX("CertAddCertificateContextToStore() failed");
+		goto error;
+	}
 
 	// Get the selected provider name and save it in the registry
 	if (providerName != NULL) wincrypto_setregistry(L"KeyStore", providerName);
-
 	goto end;
 
 error:

microscript/ILibDuktape_ScriptContainer.c

@@ -178,6 +178,7 @@ extern void ILibDuktape_MemoryStream_Init(duk_context *ctx);
 extern void ILibDuktape_NetworkMonitor_Init(duk_context *ctx);
 extern int GenerateSHA384FileHash(char *filePath, char *fileHash);
 char g_AgentCrashID[280];
+char g_AgentCrashID_HASH[17] = { 0 };
 
 typedef enum SCRIPT_ENGINE_COMMAND
 {
@@ -345,6 +346,8 @@ void ILibDuktape_ScriptContainer_CheckEmbeddedEx(char *exePath, char **script, i
 	if (tmpFile != NULL)
 	{
 		g_ILibCrashID = g_AgentCrashID;
+		g_ILibCrashID_HASH = g_AgentCrashID_HASH;
+		memcpy_s(g_AgentCrashID_HASH, sizeof(g_AgentCrashID_HASH), g_AgentCrashID + i, sizeof(g_AgentCrashID_HASH) - 1);
 
 #ifdef WIN32
 		// Read the PE Headers, to determine where to look for the Embedded JS

microstack/ILibParsers.c

@@ -70,6 +70,7 @@ limitations under the License.
 	#endif
 
 	#include <Dbghelp.h>
+	#include <fcntl.h>
 #endif
 
 #include <time.h>
@@ -2590,6 +2591,7 @@ ILibExportMethod void ILibChain_EndContinue(void *chain)
 }
 
 char* g_ILibCrashID = NULL;
+char* g_ILibCrashID_HASH = NULL;
 char* g_ILibCrashDump_path = NULL;
 
 #if defined(WIN32)
@@ -2684,7 +2686,7 @@ void ILib_WindowsExceptionDebugEx(ILib_DumpEnabledContext *dumpEnabledExceptionC
 			psym->MaxNameLen = MAX_SYM_NAME;
 			pimg->SizeOfStruct = sizeof(IMAGEHLP_LINE64);
 
-			len = sprintf_s(buffer, sizeof(buffer), "FATAL EXCEPTION [%s] @ ", (g_ILibCrashID != NULL ? g_ILibCrashID : ""));
+			len = sprintf_s(buffer, sizeof(buffer), "FATAL EXCEPTION @ ");
 #ifdef WIN64
 			len += sprintf_s(buffer + len, sizeof(buffer) - len, "[FuncAddr: 0x%016llx / BaseAddr: 0x%016llx / Delta: %lld]\n", (unsigned __int64)StackFrame.AddrPC.Offset, (unsigned __int64)&ILibCreateChain, (unsigned __int64)&ILibCreateChain - (unsigned __int64)StackFrame.AddrPC.Offset);
 #else
@@ -2731,18 +2733,6 @@ void ILib_POSIX_CrashHandler(int code)
 	{
 		memcpy_s(msgBuffer + msgLen, sizeof(msgBuffer) - msgLen, "** CRASH **\n", 12);
 		msgLen += 12;
-		if (g_ILibCrashID != NULL)
-		{
-			int idlen = strnlen_s(g_ILibCrashID, 255);
-			memcpy_s(msgBuffer + msgLen, sizeof(msgBuffer) - msgLen, "[", 1);
-			msgLen += 1;
-
-			memcpy_s(msgBuffer + msgLen, sizeof(msgBuffer) - msgLen, g_ILibCrashID, idlen);
-			msgLen += idlen;
-
-			memcpy_s(msgBuffer + msgLen, sizeof(msgBuffer) - msgLen, "]\n", 2);
-			msgLen += 2;
-		}
 	}
 	else if (code == 254)
 	{
@@ -9428,6 +9418,36 @@ void ILibGetDiskFreeSpace(void *i64FreeBytesToCaller, void *i64TotalBytes)
 #endif
 }
 
+
+
+
+int ILibFile_CopyTo(char *source, char *destination)
+{
+#ifdef WIN32
+	WCHAR SourceW[4096];
+	WCHAR DestW[4096];
+
+	ILibUTF8ToWideEx(source, -1, SourceW, (int)sizeof(SourceW) / 2);
+	ILibUTF8ToWideEx(destination, -1, DestW, (int)sizeof(DestW) / 2);
+	return(CopyFileW(SourceW, DestW, FALSE) ? 0 : 1);
+#else
+	FILE *from = fopen(source, "rb");
+	FILE *to = fopen(destination, "wb");
+	size_t bytesRead;
+	int ret = 0;
+	while ((bytesRead = fread(ILibScratchPad, 1, sizeof(ILibScratchPad), from)) > 0)
+	{
+		ret = (fwrite(ILibScratchPad, 1, bytesRead, to) > 0 ? 0 : 1);
+	}
+
+	fclose(to);
+	fclose(from);
+	return(ret);
+#endif
+	
+}
+
+
 int ILibGetMillisecondTimeSpan(struct timeval *tv1, struct timeval *tv2)
 {
 	struct timeval a;
@@ -10692,11 +10712,11 @@ char* ILibCriticalLog (const char* msg, const char* file, int line, int user1, i
 	int len = ILibGetLocalTime((char*)timeStamp, (int)sizeof(timeStamp));
 	if (file != NULL)
 	{
-		len = sprintf_s(ILibCriticalLogBuffer, sizeof(ILibCriticalLogBuffer), "\r\n[%s] %s:%d (%d,%d) %s", timeStamp, file, line, user1, user2, msg);
+		len = sprintf_s(ILibCriticalLogBuffer, sizeof(ILibCriticalLogBuffer), "\r\n[%s] [%s] %s:%d (%d,%d) %s", timeStamp, g_ILibCrashID_HASH !=NULL? g_ILibCrashID_HASH :"", file, line, user1, user2, msg);
 	}
 	else
 	{
-		len = sprintf_s(ILibCriticalLogBuffer, sizeof(ILibCriticalLogBuffer), "\r\n[%s] %s", timeStamp, msg);
+		len = sprintf_s(ILibCriticalLogBuffer, sizeof(ILibCriticalLogBuffer), "\r\n[%s] [%s] %s", timeStamp, g_ILibCrashID_HASH != NULL ? g_ILibCrashID_HASH : "", msg);
 	}
 	if (len > 0 && len < (int)sizeof(ILibCriticalLogBuffer) && ILibCriticalLogFilename != NULL) ILibAppendStringToDiskEx(ILibCriticalLogFilename, ILibCriticalLogBuffer, len);
 	if (file != NULL)

microstack/ILibParsers.h

@@ -930,7 +930,7 @@ int ILibIsRunningOnChainThread(void* chain);
 	void ILibWriteStringToDiskEx(char *FileName, char *data, int dataLen);
 	void ILibDeleteFileFromDisk(char *FileName);
 	void ILibGetDiskFreeSpace(void *i64FreeBytesToCaller, void *i64TotalBytes);
-
+	int ILibFile_CopyTo(char *source, char *destination);
 
 	/*! \defgroup StackGroup Stack
 	\ingroup DataStructures
@@ -1572,6 +1572,7 @@ int ILibIsRunningOnChainThread(void* chain);
 	void ILibChain_DebugOffset(char *buffer, int bufferLen, uint64_t addrOffset);
 	char* ILibChain_Debug(void *chain, char* buffer, int bufferLen);
 	extern char* g_ILibCrashID;
+	extern char* g_ILibCrashID_HASH;
 	extern char* g_ILibCrashDump_path;
 
 #if defined(WIN32)

microstack/ILibSimpleDataStore.c

@@ -274,12 +274,13 @@ ILibSimpleDataStore_RecordHeader_NG* ILibSimpleDataStore_ReadNextRecord(ILibSimp
 
 	ILibSimpleDataStore_RecordHeader_NG *node;
 	size_t nodeSize;
+	uint64_t currentOffset;
 
 	if (root == NULL) return NULL;
 	node = (ILibSimpleDataStore_RecordHeader_NG*)(root->scratchPad + sizeof(uint64_t));
 
 	// If the current position is the end of the file, exit now.
-	if (ILibSimpleDataStore_GetPosition(root->dataFile) == root->fileSize) return NULL;
+	if ((currentOffset = ILibSimpleDataStore_GetPosition(root->dataFile)) == root->fileSize) return NULL;
 
 	// Read sizeof(ILibSimpleDataStore_RecordNode) bytes to get record Size
 	switch (legacySize)
@@ -296,8 +297,11 @@ ILibSimpleDataStore_RecordHeader_NG* ILibSimpleDataStore_ReadNextRecord(ILibSimp
 	}
 
 	i = (int)fread((void*)node, 1, nodeSize, root->dataFile);
-	if (i < (int)nodeSize) return NULL;
-
+	if (i < (int)nodeSize)
+	{
+		ILibSimpleDataStore_SeekPosition(root->dataFile, currentOffset, SEEK_SET);
+		return NULL;
+	}
 
 	// Correct the struct, valueHash stays the same
 	node->nodeSize = (int)ntohl(node->nodeSize);
@@ -308,13 +312,17 @@ ILibSimpleDataStore_RecordHeader_NG* ILibSimpleDataStore_ReadNextRecord(ILibSimp
 	if (node->keyLen > (int)((sizeof(ILibScratchPad) - nodeSize - sizeof(uint64_t))))
 	{
 		// Invalid record
+		ILibSimpleDataStore_SeekPosition(root->dataFile, currentOffset, SEEK_SET);
 		return(NULL);
 	}
 
 	// Read the key name
 	i = (int)fread((char*)node + nodeSize, 1, node->keyLen, root->dataFile);
-	if (i != node->keyLen) return NULL; // Reading Key Failed
-
+	if (i != node->keyLen)
+	{
+		ILibSimpleDataStore_SeekPosition(root->dataFile, currentOffset, SEEK_SET);
+		return NULL; // Reading Key Failed
+	}
 	// Validate Data, in 4k chunks at a time
 	bytesLeft = node->valueLength;
 
@@ -342,6 +350,7 @@ ILibSimpleDataStore_RecordHeader_NG* ILibSimpleDataStore_ReadNextRecord(ILibSimp
 			}
 		}
 
+		ILibSimpleDataStore_SeekPosition(root->dataFile, currentOffset, SEEK_SET);
 		return NULL; // Data is corrupt
 	}
 	return node;
@@ -365,12 +374,14 @@ void ILibSimpleDataStore_RebuildKeyTable(ILibSimpleDataStore_Root *root)
 	ILibSimpleDataStore_RecordHeader_NG *node = NULL;
 	ILibSimpleDataStore_TableEntry *entry;
 	int count;
+	uint64_t newoffset;
 
 	if (root == NULL) return;
 
 	ILibHashtable_ClearEx(root->keyTable, ILibSimpleDataStore_TableClear_Sink, root); // Wipe the key table, we will rebulit it
+	fseek(root->dataFile, 0, SEEK_END); // See the start of the file
+	root->fileSize = ILibSimpleDataStore_GetPosition(root->dataFile);
 	fseek(root->dataFile, 0, SEEK_SET); // See the start of the file
-	root->fileSize = -1; // Indicate we can't write to the data store
 
 
 	// First, try NG Format
@@ -475,7 +486,19 @@ void ILibSimpleDataStore_RebuildKeyTable(ILibSimpleDataStore_Root *root)
 	else
 	{
 		// No need to convert db format, because we're already NG format
-		root->fileSize = ILibSimpleDataStore_GetPosition(root->dataFile);
+		if ((newoffset = ILibSimpleDataStore_GetPosition(root->dataFile)) != root->fileSize)
+		{
+			// DB corruption detected
+			ILIBLOGMESSAGEX("DB Corruption Detected");
+			char *dest = ILibString_Replace(root->filePath, strnlen_s(root->filePath, sizeof(ILibScratchPad)), ".db", 3, ".corrupt.db", 11);
+			ILibFile_CopyTo(root->filePath, dest);
+			free(dest);
+#ifdef WIN32
+			_chsize_s(_fileno(root->dataFile), newoffset);
+#else
+			ftruncate(fileno(root->dataFile), newoffset);
+#endif
+		}
 	}
 }