using Bit.Seeder.Migration.Models; using Bit.Seeder.Migration.Utils; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging; namespace Bit.Seeder.Migration.Databases; /// /// SQLite database importer that handles schema creation and data import. /// public class SqliteImporter(DatabaseConfig config, ILogger logger) : IDatabaseImporter { private readonly ILogger _logger = logger; private readonly string _databasePath = config.Database; private SqliteConnection? _connection; private bool _disposed = false; public bool Connect() { try { // Ensure directory exists var directory = Path.GetDirectoryName(_databasePath); if (!string.IsNullOrEmpty(directory)) { Directory.CreateDirectory(directory); } var connectionString = $"Data Source={_databasePath}"; _connection = new SqliteConnection(connectionString); _connection.Open(); // Enable foreign keys and set pragmas for better performance using (var command = new SqliteCommand("PRAGMA foreign_keys = ON", _connection)) { command.ExecuteNonQuery(); } using (var command = new SqliteCommand("PRAGMA journal_mode = WAL", _connection)) { command.ExecuteNonQuery(); } using (var command = new SqliteCommand("PRAGMA synchronous = NORMAL", _connection)) { command.ExecuteNonQuery(); } _logger.LogInformation("Connected to SQLite database: {DatabasePath}", _databasePath); return true; } catch (Exception ex) { _logger.LogError("Failed to connect to SQLite: {Message}", ex.Message); return false; } } public void Disconnect() { if (_connection != null) { try { // Force completion of any pending WAL operations using (var command = new SqliteCommand("PRAGMA wal_checkpoint(TRUNCATE)", _connection)) { command.ExecuteNonQuery(); } } catch (Exception ex) { _logger.LogWarning("Error during WAL checkpoint: {Message}", ex.Message); } _connection.Close(); _connection.Dispose(); _connection = null; _logger.LogInformation("Disconnected from SQLite"); } } /// /// Creates a table from the provided schema definition. /// public bool CreateTableFromSchema( string tableName, List columns, Dictionary columnTypes, List? specialColumns = null) { specialColumns ??= []; if (_connection == null) throw new InvalidOperationException("Not connected to database"); // Validate table name to prevent SQL injection IdentifierValidator.ValidateOrThrow(tableName, "table name"); try { var sqliteColumns = new List(); foreach (var colName in columns) { // Validate each column name to prevent SQL injection IdentifierValidator.ValidateOrThrow(colName, "column name"); var sqlServerType = columnTypes.GetValueOrDefault(colName, "VARCHAR(MAX)"); var sqliteType = ConvertSqlServerTypeToSQLite(sqlServerType, specialColumns.Contains(colName)); sqliteColumns.Add($"\"{colName}\" {sqliteType}"); } var createSql = $@" CREATE TABLE IF NOT EXISTS ""{tableName}"" ( {string.Join(",\n ", sqliteColumns)} )"; _logger.LogInformation("Creating table {TableName} in SQLite", tableName); _logger.LogDebug("CREATE TABLE SQL: {CreateSql}", createSql); using var command = new SqliteCommand(createSql, _connection); command.ExecuteNonQuery(); _logger.LogInformation("Successfully created table {TableName}", tableName); return true; } catch (Exception ex) { _logger.LogError("Error creating table {TableName}: {Message}", tableName, ex.Message); return false; } } /// /// Gets the list of columns for a table. /// public List GetTableColumns(string tableName) { if (_connection == null) throw new InvalidOperationException("Not connected to database"); // Validate table name to prevent SQL injection IdentifierValidator.ValidateOrThrow(tableName, "table name"); try { var query = $"PRAGMA table_info(\"{tableName}\")"; using var command = new SqliteCommand(query, _connection); using var reader = command.ExecuteReader(); var columns = new List(); while (reader.Read()) { columns.Add(reader.GetString(1)); // Column name is at index 1 } return columns; } catch (Exception ex) { _logger.LogError("Error getting columns for table {TableName}: {Message}", tableName, ex.Message); return []; } } /// /// Imports data into a table using batch INSERT statements. /// public bool ImportData( string tableName, List columns, List data, int batchSize = DbSeederConstants.DEFAULT_BATCH_SIZE) { if (_connection == null) throw new InvalidOperationException("Not connected to database"); // Validate table name to prevent SQL injection IdentifierValidator.ValidateOrThrow(tableName, "table name"); if (data.Count == 0) { _logger.LogWarning("No data to import for table {TableName}", tableName); return true; } try { var actualColumns = GetTableColumns(tableName); if (actualColumns.Count == 0) { _logger.LogError("Could not retrieve columns for table {TableName}", tableName); return false; } // Filter columns var validColumnIndices = new List(); var validColumns = new List(); for (int i = 0; i < columns.Count; i++) { if (actualColumns.Contains(columns[i])) { // Validate column name to prevent SQL injection IdentifierValidator.ValidateOrThrow(columns[i], "column name"); validColumnIndices.Add(i); validColumns.Add(columns[i]); } } if (validColumns.Count == 0) { _logger.LogError("No valid columns found for table {TableName}", tableName); return false; } var filteredData = data.Select(row => validColumnIndices.Select(i => i < row.Length ? row[i] : null).ToArray() ).ToList(); _logger.LogInformation("Importing {Count} rows into {TableName}", filteredData.Count, tableName); // Build INSERT statement var quotedColumns = validColumns.Select(col => $"\"{col}\"").ToList(); var placeholders = string.Join(", ", Enumerable.Range(0, validColumns.Count).Select(i => $"@p{i}")); var insertSql = $"INSERT INTO \"{tableName}\" ({string.Join(", ", quotedColumns)}) VALUES ({placeholders})"; // Begin transaction for all batches using var transaction = _connection.BeginTransaction(); try { var totalImported = 0; for (int i = 0; i < filteredData.Count; i += batchSize) { var batch = filteredData.Skip(i).Take(batchSize).ToList(); foreach (var row in batch) { using var command = new SqliteCommand(insertSql, _connection, transaction); var preparedRow = PrepareRowForInsert(row, validColumns); for (int p = 0; p < preparedRow.Length; p++) { var value = preparedRow[p] ?? DBNull.Value; // For string values, explicitly set parameter type to avoid truncation if (value is string strValue) { var param = command.Parameters.Add($"@p{p}", Microsoft.Data.Sqlite.SqliteType.Text); param.Value = strValue; } else { command.Parameters.AddWithValue($"@p{p}", value); } } command.ExecuteNonQuery(); } totalImported += batch.Count; if (filteredData.Count > DbSeederConstants.LOGGING_THRESHOLD) { _logger.LogDebug("Batch: {BatchCount} rows ({TotalImported}/{FilteredDataCount} total)", batch.Count, totalImported, filteredData.Count); } } transaction.Commit(); _logger.LogInformation("Successfully imported {TotalImported} rows into {TableName}", totalImported, tableName); return true; } catch { // Safely rollback transaction, preserving original exception transaction.SafeRollback(_connection, _logger, tableName); throw; } } catch (Exception ex) { _logger.LogError("Error importing data into {TableName}: {Message}", tableName, ex.Message); return false; } } /// /// Checks if a table exists in the database. /// public bool TableExists(string tableName) { if (_connection == null) throw new InvalidOperationException("Not connected to database"); // Validate table name to prevent SQL injection IdentifierValidator.ValidateOrThrow(tableName, "table name"); try { var query = "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name = @tableName"; using var command = new SqliteCommand(query, _connection); command.Parameters.AddWithValue("@tableName", tableName); // Use null-safe scalar value retrieval var count = command.GetScalarValue(0, _logger, $"table existence check for {tableName}"); return count > 0; } catch (Exception ex) { _logger.LogError("Error checking if table {TableName} exists: {Message}", tableName, ex.Message); return false; } } /// /// Gets the row count for a specific table. /// public int GetTableRowCount(string tableName) { if (_connection == null) throw new InvalidOperationException("Not connected to database"); // Validate table name to prevent SQL injection IdentifierValidator.ValidateOrThrow(tableName, "table name"); try { var query = $"SELECT COUNT(*) FROM \"{tableName}\""; using var command = new SqliteCommand(query, _connection); // Use null-safe scalar value retrieval return command.GetScalarValue(0, _logger, $"row count for {tableName}"); } catch (Exception ex) { _logger.LogError("Error getting row count for {TableName}: {Message}", tableName, ex.Message); return 0; } } /// /// Drops a table from the database. /// public bool DropTable(string tableName) { if (_connection == null) throw new InvalidOperationException("Not connected to database"); // Validate table name to prevent SQL injection IdentifierValidator.ValidateOrThrow(tableName, "table name"); try { var query = $"DROP TABLE IF EXISTS \"{tableName}\""; using var command = new SqliteCommand(query, _connection); command.ExecuteNonQuery(); _logger.LogInformation("Dropped table {TableName}", tableName); return true; } catch (Exception ex) { _logger.LogError("Error dropping table {TableName}: {Message}", tableName, ex.Message); return false; } } public bool DisableForeignKeys() { if (_connection == null) throw new InvalidOperationException("Not connected to database"); try { _logger.LogInformation("Disabling foreign key constraints"); var query = "PRAGMA foreign_keys = OFF"; using var command = new SqliteCommand(query, _connection); command.ExecuteNonQuery(); _logger.LogInformation("Foreign key constraints disabled"); return true; } catch (Exception ex) { _logger.LogError("Error disabling foreign key constraints: {Message}", ex.Message); return false; } } public bool EnableForeignKeys() { if (_connection == null) throw new InvalidOperationException("Not connected to database"); try { _logger.LogInformation("Re-enabling foreign key constraints"); var query = "PRAGMA foreign_keys = ON"; using var command = new SqliteCommand(query, _connection); command.ExecuteNonQuery(); _logger.LogInformation("Foreign key constraints re-enabled"); return true; } catch (Exception ex) { _logger.LogError("Error re-enabling foreign key constraints: {Message}", ex.Message); return false; } } private string ConvertSqlServerTypeToSQLite(string sqlServerType, bool isJsonColumn) { var baseType = sqlServerType.Replace(" NULL", "").Replace(" NOT NULL", "").Trim().ToUpper(); var isNullable = !sqlServerType.Contains("NOT NULL"); if (isJsonColumn) return "TEXT" + (isNullable ? "" : " NOT NULL"); // SQLite has only 5 storage classes: NULL, INTEGER, REAL, TEXT, BLOB string sqliteType; if (baseType.Contains("INT") || baseType.Contains("BIT")) sqliteType = "INTEGER"; else if (baseType.Contains("DECIMAL") || baseType.Contains("NUMERIC") || baseType.Contains("FLOAT") || baseType.Contains("REAL") || baseType.Contains("MONEY")) sqliteType = "REAL"; else if (baseType.Contains("BINARY") || baseType == "IMAGE") sqliteType = "BLOB"; else sqliteType = "TEXT"; return sqliteType + (isNullable ? "" : " NOT NULL"); } private object[] PrepareRowForInsert(object?[] row, List columns) { return row.Select(value => { if (value == null || value == DBNull.Value) return DBNull.Value; if (value is string strValue) { // Only convert truly empty strings to DBNull, not whitespace // This preserves JSON strings and other data that might have whitespace if (strValue.Length == 0) return DBNull.Value; // Boolean to integer for SQLite if (strValue.Equals("true", StringComparison.OrdinalIgnoreCase)) return 1; if (strValue.Equals("false", StringComparison.OrdinalIgnoreCase)) return 0; // Handle datetime with timezone if ((strValue.Contains('+') || strValue.EndsWith('Z')) && DateTimeHelper.IsLikelyIsoDateTime(strValue)) { return DateTimeHelper.RemoveTimezone(strValue) ?? strValue; } } return value; }).ToArray(); } public bool SupportsBulkCopy() { // SQLite performs better with the original row-by-row INSERT approach // Multi-row INSERT causes performance degradation for SQLite return false; } public bool ImportDataBulk( string tableName, List columns, List data) { // Not implemented for SQLite - use standard ImportData instead _logger.LogWarning("Bulk copy not supported for SQLite, use standard import"); return false; } /// /// Tests the connection to SQLite by executing a simple query. /// public bool TestConnection() { try { if (Connect()) { using var command = new SqliteCommand("SELECT 1", _connection); // Use null-safe scalar value retrieval var result = command.GetScalarValue(0, _logger, "connection test"); Disconnect(); return result == 1; } return false; } catch (Exception ex) { _logger.LogError("SQLite connection test failed: {Message}", ex.Message); return false; } } /// /// Disposes of the SQLite importer and releases all resources. /// public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } /// /// Protected implementation of Dispose pattern. /// /// True if called from Dispose(), false if called from finalizer protected virtual void Dispose(bool disposing) { if (!_disposed) { if (disposing) { // Dispose managed resources Disconnect(); } _disposed = true; } } }