diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 07f9917a5e..88cfc71256 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,7 +14,7 @@ .github/workflows/publish.yml @bitwarden/dept-bre ## These are shared workflows ## -.github/workflows/_move_finalization_db_scripts.yml +.github/workflows/_move_edd_db_scripts.yml .github/workflows/release.yml # Database Operations for database changes diff --git a/.github/workflows/_move_finalization_db_scripts.yml b/.github/workflows/_move_edd_db_scripts.yml similarity index 66% rename from .github/workflows/_move_finalization_db_scripts.yml rename to .github/workflows/_move_edd_db_scripts.yml index 33d828fef7..98fe4f1f05 100644 --- a/.github/workflows/_move_finalization_db_scripts.yml +++ b/.github/workflows/_move_edd_db_scripts.yml @@ -1,5 +1,5 @@ -name: _move_finalization_db_scripts -run-name: Move finalization database scripts +name: _move_edd_db_scripts +run-name: Move EDD database scripts on: workflow_call: @@ -17,7 +17,8 @@ jobs: id-token: write outputs: migration_filename_prefix: ${{ steps.prefix.outputs.prefix }} - copy_finalization_scripts: ${{ steps.check-finalization-scripts-existence.outputs.copy_finalization_scripts }} + copy_edd_scripts: ${{ steps.check-script-existence.outputs.copy_edd_scripts }} + steps: - name: Log in to Azure uses: bitwarden/gh-actions/azure-login@main @@ -45,17 +46,17 @@ jobs: id: prefix run: echo "prefix=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - name: Check if any files in DB finalization directory - id: check-finalization-scripts-existence + - name: Check if any files in DB transition or finalization directories + id: check-script-existence run: | - if [ -f util/Migrator/DbScripts_finalization/* ]; then - echo "copy_finalization_scripts=true" >> $GITHUB_OUTPUT + if [ -f util/Migrator/DbScripts_transition/* -o -f util/Migrator/DbScripts_finalization/* ]; then + echo "copy_edd_scripts=true" >> $GITHUB_OUTPUT else - echo "copy_finalization_scripts=false" >> $GITHUB_OUTPUT + echo "copy_edd_scripts=false" >> $GITHUB_OUTPUT fi - move-finalization-db-scripts: - name: Move finalization database scripts + move-scripts: + name: Move scripts runs-on: ubuntu-22.04 needs: setup permissions: @@ -63,9 +64,9 @@ jobs: pull-requests: write id-token: write actions: read - if: ${{ needs.setup.outputs.copy_finalization_scripts == 'true' }} + if: ${{ needs.setup.outputs.copy_edd_scripts == 'true' }} steps: - - name: Checkout + - name: Check out repo uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 @@ -74,35 +75,62 @@ jobs: id: branch_name env: PREFIX: ${{ needs.setup.outputs.migration_filename_prefix }} - run: echo "branch_name=move_finalization_db_scripts_$PREFIX" >> $GITHUB_OUTPUT + run: echo "branch_name=move_edd_db_scripts_$PREFIX" >> $GITHUB_OUTPUT - name: "Create branch" env: BRANCH: ${{ steps.branch_name.outputs.branch_name }} run: git switch -c $BRANCH - - name: Move DbScripts_finalization + - name: Move scripts and finalization database schema id: move-files env: PREFIX: ${{ needs.setup.outputs.migration_filename_prefix }} run: | - src_dir="util/Migrator/DbScripts_finalization" + # scripts + moved_files="Migration scripts moved:\n\n" + + src_dirs="util/Migrator/DbScripts_transition,util/Migrator/DbScripts_finalization" dest_dir="util/Migrator/DbScripts" i=0 - moved_files="" - for file in "$src_dir"/*; do - filenumber=$(printf "%02d" $i) + for src_dir in ${src_dirs//,/ }; do + for file in "$src_dir"/*; do + filenumber=$(printf "%02d" $i) - filename=$(basename "$file") - new_filename="${PREFIX}_${filenumber}_${filename}" - dest_file="$dest_dir/$new_filename" + filename=$(basename "$file") + new_filename="${PREFIX}_${filenumber}_${filename}" + dest_file="$dest_dir/$new_filename" - mv "$file" "$dest_file" - moved_files="$moved_files \n $filename -> $new_filename" + # Replace any finalization references due to the move + sed -i -e 's/dbo_finalization/dbo/g' "$file" - i=$((i+1)) + mv "$file" "$dest_file" + moved_files="$moved_files \n $filename -> $new_filename" + + i=$((i+1)) + done done + + # schema + moved_files="$moved_files\n\nFinalization scripts moved:\n\n" + + src_dir="src/Sql/dbo_finalization" + dest_dir="src/Sql/dbo" + + # sync finalization schema back to dbo, maintaining structure + rsync -r "$src_dir/" "$dest_dir/" + rm -rf $src_dir/* + + # Replace any finalization references due to the move + find ./src/Sql/dbo -name "*.sql" -type f -exec sed -i \ + -e 's/\[dbo_finalization\]/[dbo]/g' \ + -e 's/dbo_finalization\./dbo./g' {} + + + for file in "$src_dir"/**/*; do + moved_files="$moved_files \n $file" + done + echo "moved_files=$moved_files" >> $GITHUB_OUTPUT - name: Log in to Azure @@ -139,7 +167,7 @@ jobs: git config --local user.name "bitwarden-devops-bot" if [ -n "$(git status --porcelain)" ]; then git add . - git commit -m "Move DbScripts_finalization to DbScripts" -a + git commit -m "Move EDD database scripts" -a git push -u origin ${{ steps.branch_name.outputs.branch_name }} echo "pr_needed=true" >> $GITHUB_OUTPUT else @@ -155,16 +183,16 @@ jobs: BRANCH: ${{ steps.branch_name.outputs.branch_name }} GH_TOKEN: ${{ github.token }} MOVED_FILES: ${{ steps.move-files.outputs.moved_files }} - TITLE: "Move finalization database scripts" + TITLE: "Move EDD database scripts" run: | PR_URL=$(gh pr create --title "$TITLE" \ --base "main" \ --head "$BRANCH" \ --label "automated pr" \ --body " - ## Automated movement of DbScripts_finalization to DbScripts + Automated movement of EDD database scripts. - ## Files moved: + Files moved: $(echo -e "$MOVED_FILES") ") echo "pr_url=${PR_URL}" >> $GITHUB_OUTPUT @@ -175,5 +203,5 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ steps.retrieve-secrets.outputs.devops-alerts-slack-webhook-url }} with: - message: "Created PR for moving DbScripts_finalization to DbScripts: ${{ steps.create-pr.outputs.pr_url }}" + message: "Created PR for moving EDD database scripts: ${{ steps.create-pr.outputs.pr_url }}" status: ${{ job.status }} diff --git a/.github/workflows/repository-management.yml b/.github/workflows/repository-management.yml index 2d75cbcb4a..b5d6db69d4 100644 --- a/.github/workflows/repository-management.yml +++ b/.github/workflows/repository-management.yml @@ -228,8 +228,8 @@ jobs: git switch --quiet --create $BRANCH_NAME git push --quiet --set-upstream origin $BRANCH_NAME - move_future_db_scripts: - name: Move finalization database scripts + move_edd_db_scripts: + name: Move EDD database scripts needs: cut_branch - uses: ./.github/workflows/_move_finalization_db_scripts.yml + uses: ./.github/workflows/_move_edd_db_scripts.yml secrets: inherit diff --git a/src/Sql/Sql.sqlproj b/src/Sql/Sql.sqlproj index 849fd3bdfd..1a7530321e 100644 --- a/src/Sql/Sql.sqlproj +++ b/src/Sql/Sql.sqlproj @@ -11,8 +11,6 @@ - - diff --git a/src/Sql/dbo_finalization/.gitkeep b/src/Sql/dbo_finalization/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/util/Migrator/Migrator.csproj b/util/Migrator/Migrator.csproj index b425babea3..bef6dadfb8 100644 --- a/util/Migrator/Migrator.csproj +++ b/util/Migrator/Migrator.csproj @@ -3,6 +3,7 @@ + diff --git a/util/Migrator/MigratorConstants.cs b/util/Migrator/MigratorConstants.cs index 1ffcfcfe22..bf1bccb6c2 100644 --- a/util/Migrator/MigratorConstants.cs +++ b/util/Migrator/MigratorConstants.cs @@ -4,5 +4,5 @@ public static class MigratorConstants { public const string SqlTableJournalName = "Migration"; public const string DefaultMigrationsFolderName = "DbScripts"; - public const string TransitionMigrationsFolderName = "DbScripts_data_migration"; + public const string TransitionMigrationsFolderName = "DbScripts_transition"; } diff --git a/util/Migrator/README.md b/util/Migrator/README.md new file mode 100644 index 0000000000..950c2b682e --- /dev/null +++ b/util/Migrator/README.md @@ -0,0 +1,7 @@ +# Bitwarden Database Migrator + +A class library leveraged by [utilities](../MsSqlMigratorUtility) and [hosted applications](/src/Admin/HostedServices/DatabaseMigrationHostedService.cs) to perform SQL database migrations. A [MSSQL migrator](./SqlServerDbMigrator.cs) exists here as the default use case. + +In production environments the Migrator is typically executed during application startup or as part of CI/CD pipelines to ensure database schemas are up-to-date before application deployment. + +See the [documentation on creating migrations](https://contributing.bitwarden.com/contributing/database-migrations/) for how to utilize the files seen here. diff --git a/util/MsSqlMigratorUtility/README.md b/util/MsSqlMigratorUtility/README.md new file mode 100644 index 0000000000..469d1bb43e --- /dev/null +++ b/util/MsSqlMigratorUtility/README.md @@ -0,0 +1,16 @@ +# Bitwarden MSSQL Database Migrator Utility + +A command-line utility for performing MSSQL database migrations for Bitwarden's self-hosted and cloud deployments. + +## Overview + +The MSSQL Migrator Utility is a specialized tool that leverages the [Migrator library](../Migrator) to handle MSSQL database migrations. The utility uses [DbUp](https://dbup.github.io/) to handle the execution and tracking of database migrations. It runs SQL scripts in order, tracking which scripts have been executed to avoid duplicate runs. + +## Features + +- Command-line interface for executing database migrations +- Integration with DbUp for reliable migration management +- Execution inside or outside of transactions for different application scenarios +- Script execution tracking to prevent duplicate migrations and support retries + +See the [documentation](https://contributing.bitwarden.com/getting-started/server/database/mssql/#updating-the-database) for usage. diff --git a/util/MySqlMigrations/README.md b/util/MySqlMigrations/README.md new file mode 100644 index 0000000000..3373aabaee --- /dev/null +++ b/util/MySqlMigrations/README.md @@ -0,0 +1,5 @@ +# Bitwarden MySQL Database Migrator + +A class library leveraged by [hosted applications](/src/Admin/HostedServices/DatabaseMigrationHostedService.cs) to perform MySQL database migrations via Entity Framework. + +See the [documentation on creating migrations](https://contributing.bitwarden.com/contributing/database-migrations/) for how to utilize the files seen here. diff --git a/util/PostgresMigrations/README.md b/util/PostgresMigrations/README.md new file mode 100644 index 0000000000..4436fbe72f --- /dev/null +++ b/util/PostgresMigrations/README.md @@ -0,0 +1,5 @@ +# Bitwarden PostgreSQL Database Migrator + +A class library leveraged by [hosted applications](/src/Admin/HostedServices/DatabaseMigrationHostedService.cs) to perform PostgreSQL database migrations via Entity Framework. + +See the [documentation on creating migrations](https://contributing.bitwarden.com/contributing/database-migrations/) for how to utilize the files seen here. diff --git a/util/SqliteMigrations/README.md b/util/SqliteMigrations/README.md new file mode 100644 index 0000000000..ff55a7c1b7 --- /dev/null +++ b/util/SqliteMigrations/README.md @@ -0,0 +1,5 @@ +# Bitwarden SQLite Database Migrator + +A class library leveraged by [hosted applications](/src/Admin/HostedServices/DatabaseMigrationHostedService.cs) to perform SQLite database migrations via Entity Framework. + +See the [documentation on creating migrations](https://contributing.bitwarden.com/contributing/database-migrations/) for how to utilize the files seen here.