From c33b020f567a3e91413c11bfe91a1c230c534f94 Mon Sep 17 00:00:00 2001 From: Emmzyemms Date: Thu, 27 Feb 2025 17:01:16 +0100 Subject: [PATCH] db-migration --- .../db-migration/commands/backup.commands.ts | 226 ++++++++ .../commands/migration.commands.ts | 226 ++++++++ .../commands/rollback.commands.ts | 110 ++++ .../db-migration/commands/test.commands.ts | 55 ++ .../db-migration/db-migration.constants.ts | 4 + .../src/db-migration/db-migration.module.ts | 222 ++++++++ .../entities/backup-metadata.entity.ts | 35 ++ .../entities/migration-history.entity.ts | 49 ++ .../entities/migration-lock.entity.ts | 26 + .../interfaces/backup.interface.ts | 87 +++ .../interfaces/migration.interface.ts | 53 ++ .../interfaces/validator.interface.ts | 29 + .../db-migration/services/backup.service.ts | 512 ++++++++++++++++++ .../services/migration.service.ts | 463 ++++++++++++++++ .../db-migration/services/rollback.service.ts | 181 +++++++ .../db-migration/services/testing.service.ts | 209 +++++++ .../services/validator.service.ts | 248 +++++++++ 17 files changed, 2735 insertions(+) create mode 100644 backend/src/db-migration/commands/backup.commands.ts create mode 100644 backend/src/db-migration/commands/migration.commands.ts create mode 100644 backend/src/db-migration/commands/rollback.commands.ts create mode 100644 backend/src/db-migration/commands/test.commands.ts create mode 100644 backend/src/db-migration/db-migration.constants.ts create mode 100644 backend/src/db-migration/db-migration.module.ts create mode 100644 backend/src/db-migration/entities/backup-metadata.entity.ts create mode 100644 backend/src/db-migration/entities/migration-history.entity.ts create mode 100644 backend/src/db-migration/entities/migration-lock.entity.ts create mode 100644 backend/src/db-migration/interfaces/backup.interface.ts create mode 100644 backend/src/db-migration/interfaces/migration.interface.ts create mode 100644 backend/src/db-migration/interfaces/validator.interface.ts create mode 100644 backend/src/db-migration/services/backup.service.ts create mode 100644 backend/src/db-migration/services/migration.service.ts create mode 100644 backend/src/db-migration/services/rollback.service.ts create mode 100644 backend/src/db-migration/services/testing.service.ts create mode 100644 backend/src/db-migration/services/validator.service.ts diff --git a/backend/src/db-migration/commands/backup.commands.ts b/backend/src/db-migration/commands/backup.commands.ts new file mode 100644 index 00000000..d61b5b89 --- /dev/null +++ b/backend/src/db-migration/commands/backup.commands.ts @@ -0,0 +1,226 @@ +// src/db-migration/commands/backup.commands.ts +import { Command, CommandRunner, Option } from 'nest-commander'; +import { BackupService } from '../services/backup.service'; + +@Command({ name: 'backup:create', description: 'Create a database backup' }) +export class BackupCreateCommand extends CommandRunner { + constructor(private readonly backupService: BackupService) { + super(); + } + + @Option({ + flags: '-p, --provider ', + description: 'Backup provider to use', + }) + parseProvider(val: string): string { + return val; + } + + @Option({ + flags: '-r, --reason ', + description: 'Reason for the backup', + }) + parseReason(val: string): string { + return val; + } + + async run( + passedParams: string[], + options?: Record, + ): Promise { + try { + console.log('Creating database backup...'); + + const result = await this.backupService.createBackup({ + provider: options?.provider, + reason: options?.reason || 'manual-cli', + }); + + if (result.success) { + console.log(`Backup created successfully with ID: ${result.backupId}`); + console.log(`Size: ${this.formatSize(result.size || 0)}`); + console.log(`Location: ${result.location}`); + } else { + console.error(`Backup failed: ${result.error}`); + process.exit(1); + } + } catch (error) { + console.error(`Error: ${error.message}`); + process.exit(1); + } + } + + private formatSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; + } +} + +@Command({ name: 'backup:restore', description: 'Restore a database from backup' }) +export class BackupRestoreCommand extends CommandRunner { + constructor(private readonly backupService: BackupService) { + super(); + } + + @Option({ + flags: '-p, --provider ', + description: 'Backup provider to use', + }) + parseProvider(val: string): string { + return val; + } + + async run( + passedParams: string[], + options?: Record, + ): Promise { + try { + if (passedParams.length === 0) { + console.error('Error: Backup ID is required'); + process.exit(1); + } + + const backupId = passedParams[0]; + + console.log(`Restoring database from backup ${backupId}...`); + + const result = await this.backupService.restoreBackup(backupId, { + provider: options?.provider, + }); + + if (result) { + console.log(`Database successfully restored from backup ${backupId}`); + } else { + console.error(`Failed to restore database from backup ${backupId}`); + process.exit(1); + } + } catch (error) { + console.error(`Error: ${error.message}`); + process.exit(1); + } + } +} + +@Command({ name: 'backup:list', description: 'List available backups' }) +export class BackupListCommand extends CommandRunner { + constructor(private readonly backupService: BackupService) { + super(); + } + + @Option({ + flags: '-p, --provider ', + description: 'Filter by backup provider', + }) + parseProvider(val: string): string { + return val; + } + + @Option({ + flags: '-m, --migration ', + description: 'Filter by migration ID', + }) + parseMigrationId(val: string): string { + return val; + } + + @Option({ + flags: '-l, --limit ', + description: 'Limit number of results', + }) + parseLimit(val: string): number { + return parseInt(val, 10); + } + + async run( + passedParams: string[], + options?: Record, + ): Promise { + try { + const backups = await this.backupService.listBackups({ + provider: options?.provider, + migrationId: options?.migration, + limit: options?.limit, + }); + + if (backups.length === 0) { + console.log('No backups found'); + return; + } + + console.log(`Found ${backups.length} backups:`); + + for (const backup of backups) { + console.log(`- ID: ${backup.backupId}`); + console.log(` Timestamp: ${new Date(backup.timestamp).toISOString()}`); + console.log(` Size: ${this.formatSize(backup.size)}`); + + if (backup.migrationId) { + console.log(` Migration: ${backup.migrationId}`); + } + + if (backup.metadata?.reason) { + console.log(` Reason: ${backup.metadata.reason}`); + } + + console.log(''); + } + } catch (error) { + console.error(`Error: ${error.message}`); + process.exit(1); + } + } + + private formatSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; + } +} + +@Command({ name: 'backup:delete', description: 'Delete a backup' }) +export class BackupDeleteCommand extends CommandRunner { + constructor(private readonly backupService: BackupService) { + super(); + } + + @Option({ + flags: '-p, --provider ', + description: 'Backup provider to use', + }) + parseProvider(val: string): string { + return val; + } + + async run( + passedParams: string[], + options?: Record, + ): Promise { + try { + if (passedParams.length === 0) { + console.error('Error: Backup ID is required'); + process.exit(1); + } + + const backupId = passedParams[0]; + + console.log(`Deleting backup ${backupId}...`); + + const result = await this.backupService.deleteBackup(backupId, { + provider: options?.provider, + }); + + if (result) { + console.log(`Backup ${backupId} deleted successfully`); + } else { + console.error(`Failed to delete backup ${backupId}`); + process.exit(1); + } + } catch (error) { + console.error(`Error: ${error.message}`); + process.exit(1); + } + } +} \ No newline at end of file diff --git a/backend/src/db-migration/commands/migration.commands.ts b/backend/src/db-migration/commands/migration.commands.ts new file mode 100644 index 00000000..b0270b98 --- /dev/null +++ b/backend/src/db-migration/commands/migration.commands.ts @@ -0,0 +1,226 @@ +// src/db-migration/commands/migration.commands.ts +import { Command, CommandRunner, Option } from 'nest-commander'; +import { MigrationService } from '../services/migration.service'; +import { TestingService } from '../services/testing.service'; + +@Command({ name: 'migration:status', description: 'Show migration status' }) +export class MigrationStatusCommand extends CommandRunner { + constructor(private readonly migrationService: MigrationService) { + super(); + } + + async run( + passedParams: string[], + options?: Record, + ): Promise { + try { + const pendingMigrations = await this.migrationService.getPendingMigrations(); + const appliedMigrations = await this.migrationService.getAppliedMigrations(); + + console.log(`\nMigration Status:`); + console.log(`- Pending migrations: ${pendingMigrations.length}`); + console.log(`- Applied migrations: ${appliedMigrations.length}`); + + if (pendingMigrations.length > 0) { + console.log(`\nPending Migrations:`); + for (const migration of pendingMigrations) { + console.log(`- ${migration.id} | ${migration.name} (${new Date(migration.timestamp).toISOString()})`); + } + } + + if (appliedMigrations.length > 0) { + console.log(`\nApplied Migrations (most recent first):`); + for (const migration of appliedMigrations) { + console.log(`- ${migration.migrationId} | ${migration.name} (${migration.appliedAt.toISOString()})`); + } + } + + console.log(''); + } catch (error) { + console.error(`Error: ${error.message}`); + process.exit(1); + } + } +} + +@Command({ name: 'migration:run', description: 'Run pending migrations' }) +export class MigrationRunCommand extends CommandRunner { + constructor(private readonly migrationService: MigrationService) { + super(); + } + + @Option({ + flags: '-s, --skip-backup', + description: 'Skip database backup before migration', + }) + parseSkipBackup(val: string): boolean { + return true; + } + + @Option({ + flags: '-v, --skip-validation', + description: 'Skip data validation', + }) + parseSkipValidation(val: string): boolean { + return true; + } + + @Option({ + flags: '-t, --skip-transaction', + description: 'Skip transaction wrapping', + }) + parseSkipTransaction(val: string): boolean { + return true; + } + + async run( + passedParams: string[], + options?: Record, + ): Promise { + try { + const pendingMigrations = await this.migrationService.getPendingMigrations(); + + if (pendingMigrations.length === 0) { + console.log('No pending migrations to run'); + return; + } + + console.log(`Running ${pendingMigrations.length} pending migrations...`); + + const results = await this.migrationService.applyPendingMigrations({ + createBackup: !options?.skipBackup, + validate: !options?.skipValidation, + transaction: !options?.skipTransaction, + }); + + const successCount = results.filter(r => r.success).length; + const failureCount = results.length - successCount; + + console.log(`\nMigration Results:`); + console.log(`- Successful: ${successCount}`); + console.log(`- Failed: ${failureCount}`); + + if (failureCount > 0) { + console.log(`\nFailed Migrations:`); + for (const result of results.filter(r => !r.success)) { + console.log(`- ${result.migrationId} | ${result.name}`); + console.log(` Error: ${result.error}`); + } + + process.exit(1); + } + } catch (error) { + console.error(`Error: ${error.message}`); + process.exit(1); + } + } +} + +@Command({ name: 'migration:apply', description: 'Apply a specific migration' }) +export class MigrationApplyCommand extends CommandRunner { + constructor(private readonly migrationService: MigrationService) { + super(); + } + + @Option({ + flags: '-s, --skip-backup', + description: 'Skip database backup before migration', + }) + parseSkipBackup(val: string): boolean { + return true; + } + + @Option({ + flags: '-v, --skip-validation', + description: 'Skip data validation', + }) + parseSkipValidation(val: string): boolean { + return true; + } + + @Option({ + flags: '-t, --skip-transaction', + description: 'Skip transaction wrapping', + }) + parseSkipTransaction(val: string): boolean { + return true; + } + + async run( + passedParams: string[], + options?: Record, + ): Promise { + try { + if (passedParams.length === 0) { + console.error('Error: Migration ID is required'); + process.exit(1); + } + + const migrationId = passedParams[0]; + + console.log(`Applying migration ${migrationId}...`); + + const result = await this.migrationService.applyMigration(migrationId, { + createBackup: !options?.skipBackup, + validate: !options?.skipValidation, + transaction: !options?.skipTransaction, + }); + + if (result.success) { + console.log(`Migration ${migrationId} applied successfully`); + } else { + console.error(`Migration ${migrationId} failed: ${result.error}`); + process.exit(1); + } + } catch (error) { + console.error(`Error: ${error.message}`); + process.exit(1); + } + } +} + +@Command({ name: 'migration:test', description: 'Test a migration' }) +export class MigrationTestCommand extends CommandRunner { + constructor(private readonly testingService: TestingService) { + super(); + } + + async run( + passedParams: string[], + options?: Record, + ): Promise { + try { + if (passedParams.length === 0) { + console.error('Error: Migration ID is required'); + process.exit(1); + } + + const migrationId = passedParams[0]; + + console.log(`Testing migration ${migrationId}...`); + + const result = await this.testingService.testMigration(migrationId); + + console.log(`\nTest Results for ${migrationId}:`); + console.log(`- Apply (up): ${result.results.up ? 'SUCCESS' : 'FAILED'}`); + console.log(`- Revert (down): ${result.results.down ? 'SUCCESS' : 'FAILED'}`); + console.log(`- Validation: ${result.results.validation ? 'SUCCESS' : 'FAILED'}`); + + if (result.results.test !== undefined) { + console.log(`- Custom test: ${result.results.test ? 'SUCCESS' : 'FAILED'}`); + } + + if (result.errors && result.errors.length > 0) { + console.log(`\nErrors:`); + for (const error of result.errors) { + console.log(`- ${error}`); + } + + process.exit(1); + } + } catch (error) { + console.error(`Error: ${error.message}`); + process.exit(1); + } + } +} diff --git a/backend/src/db-migration/commands/rollback.commands.ts b/backend/src/db-migration/commands/rollback.commands.ts new file mode 100644 index 00000000..7dc76df3 --- /dev/null +++ b/backend/src/db-migration/commands/rollback.commands.ts @@ -0,0 +1,110 @@ +// src/db-migration/commands/rollback.commands.ts +import { Command, CommandRunner, Option } from 'nest-commander'; +import { RollbackService } from '../services/rollback.service'; + +@Command({ name: 'migration:rollback', description: 'Rollback last migration' }) +export class MigrationRollbackCommand extends CommandRunner { + constructor(private readonly rollbackService: RollbackService) { + super(); + } + + @Option({ + flags: '-b, --use-backup', + description: 'Use backup for rollback if available', + }) + parseUseBackup(val: string): boolean { + return true; + } + + @Option({ + flags: '-t, --skip-transaction', + description: 'Skip transaction wrapping', + }) + parseSkipTransaction(val: string): boolean { + return true; + } + + async run( + passedParams: string[], + options?: Record, + ): Promise { + try { + console.log('Rolling back last migration...'); + + const result = await this.rollbackService.rollbackLastMigration({ + useBackup: options?.useBackup, + transaction: !options?.skipTransaction, + }); + + if (result) { + console.log(`Successfully rolled back migration ${result.migrationId} (${result.name})`); + } else { + console.log('No migrations to roll back'); + } + } catch (error) { + console.error(`Error: ${error.message}`); + process.exit(1); + } + } +} + +@Command({ name: 'migration:rollback-to', description: 'Rollback to a specific migration' }) +export class MigrationRollbackToCommand extends CommandRunner { + constructor(private readonly rollbackService: RollbackService) { + super(); + } + + @Option({ + flags: '-b, --use-backup', + description: 'Use backup for rollback if available', + }) + parseUseBackup(val: string): boolean { + return true; + } + + @Option({ + flags: '-t, --skip-transaction', + description: 'Skip transaction wrapping', + }) + parseSkipTransaction(val: string): boolean { + return true; + } + + async run( + passedParams: string[], + options?: Record, + ): Promise { + try { + if (passedParams.length === 0) { + console.error('Error: Target migration ID is required'); + process.exit(1); + } + + const targetMigrationId = passedParams[0]; + + console.log(`Rolling back to migration ${targetMigrationId}...`); + + const results = await this.rollbackService.rollbackToMigration( + targetMigrationId, + { + useBackup: options?.useBackup, + transaction: !options?.skipTransaction, + } + ); + + if (results.length === 0) { + console.log(`No migrations to roll back to ${targetMigrationId}`); + } else { + console.log(`Successfully rolled back ${results.length} migrations to ${targetMigrationId}`); + + console.log('\nRolled back migrations:'); + for (const result of results) { + console.log(`- ${result.migrationId} (${result.name})`); + } + } + } catch (error) { + console.error(`Error: ${error.message}`); + process.exit(1); + } + } +} \ No newline at end of file diff --git a/backend/src/db-migration/commands/test.commands.ts b/backend/src/db-migration/commands/test.commands.ts new file mode 100644 index 00000000..370bbbf7 --- /dev/null +++ b/backend/src/db-migration/commands/test.commands.ts @@ -0,0 +1,55 @@ +// src/db-migration/commands/test.commands.ts +import { Command, CommandRunner } from 'nest-commander'; +import { TestingService } from '../services/testing.service'; + +@Command({ name: 'migration:test-sequence', description: 'Test migrations in sequence' }) +export class MigrationTestSequenceCommand extends CommandRunner { + constructor(private readonly testingService: TestingService) { + super(); + } + + async run( + passedParams: string[], + options?: Record, + ): Promise { + try { + if (passedParams.length === 0) { + console.error('Error: At least one migration ID is required'); + process.exit(1); + } + + const migrationIds = passedParams; + + console.log(`Testing sequence of ${migrationIds.length} migrations...`); + + const result = await this.testingService.testMigrationSequence(migrationIds); + + console.log(`\nTest Results:`); + + for (const [migrationId, migrationResult] of Object.entries(result.results)) { + console.log(`\nMigration ${migrationId}:`); + console.log(`- Apply (up): ${migrationResult.up ? 'SUCCESS' : 'FAILED'}`); + console.log(`- Revert (down): ${migrationResult.down ? 'SUCCESS' : 'FAILED'}`); + console.log(`- Validation: ${migrationResult.validation ? 'SUCCESS' : 'FAILED'}`); + + if (migrationResult.test !== undefined) { + console.log(`- Custom test: ${migrationResult.test ? 'SUCCESS' : 'FAILED'}`); + } + + if (result.errors && result.errors[migrationId]) { + console.log(`\nErrors:`); + for (const error of result.errors[migrationId]) { + console.log(`- ${error}`); + } + } + } + + if (!result.success) { + process.exit(1); + } + } catch (error) { + console.error(`Error: ${error.message}`); + process.exit(1); + } + } +} diff --git a/backend/src/db-migration/db-migration.constants.ts b/backend/src/db-migration/db-migration.constants.ts new file mode 100644 index 00000000..be27a9a6 --- /dev/null +++ b/backend/src/db-migration/db-migration.constants.ts @@ -0,0 +1,4 @@ +// src/db-migration/db-migration.constants.ts +export const DB_MIGRATION_DIR = 'DB_MIGRATION_DIR'; +export const DB_MIGRATION_BACKUP_DIR = 'DB_MIGRATION_BACKUP_DIR'; +export const DB_MIGRATION_DEFAULT_PROVIDER = 'DB_MIGRATION_DEFAULT_PROVIDER'; \ No newline at end of file diff --git a/backend/src/db-migration/db-migration.module.ts b/backend/src/db-migration/db-migration.module.ts new file mode 100644 index 00000000..86e9a2d8 --- /dev/null +++ b/backend/src/db-migration/db-migration.module.ts @@ -0,0 +1,222 @@ +// src/db-migration/db-migration.module.ts +import { DynamicModule, Module, Provider } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { MigrationHistory } from './entities/migration-history.entity'; +import { MigrationLock } from './entities/migration-lock.entity'; +import { BackupMetadata } from './entities/backup-metadata.entity'; +import { MigrationService } from './services/migration.service'; +import { RollbackService } from './services/rollback.service'; +import { ValidatorService } from './services/validator.service'; +import { BackupService } from './services/backup.service'; +import { TestingService } from './services/testing.service'; +import { DB_MIGRATION_DIR, DB_MIGRATION_BACKUP_DIR, DB_MIGRATION_DEFAULT_PROVIDER } from './db-migration.constants'; +import { + MigrationStatusCommand, + MigrationRunCommand, + MigrationApplyCommand, + MigrationTestCommand +} from './commands/migration.commands'; +import { + MigrationRollbackCommand, + MigrationRollbackToCommand +} from './commands/rollback.commands'; +import { + BackupCreateCommand, + BackupRestoreCommand, + BackupListCommand, + BackupDeleteCommand +} from './commands/backup.commands'; +import { MigrationTestSequenceCommand } from './commands/test.commands'; + +interface DbMigrationModuleOptions { + migrationDir?: string; + backupDir?: string; + defaultBackupProvider?: string; + enableCli?: boolean; +} + +@Module({}) +export class DbMigrationModule { + static register(options: DbMigrationModuleOptions = {}): DynamicModule { + const { + migrationDir = './migrations', + backupDir = './backups', + defaultBackupProvider = 'filesystem', + enableCli = true, + } = options; + + const providers: Provider[] = [ + { + provide: DB_MIGRATION_DIR, + useValue: migrationDir, + }, + { + provide: DB_MIGRATION_BACKUP_DIR, + useValue: backupDir, + }, + { + provide: DB_MIGRATION_DEFAULT_PROVIDER, + useValue: defaultBackupProvider, + }, + MigrationService, + RollbackService, + ValidatorService, + BackupService, + TestingService, + ]; + + if (enableCli) { + providers.push( + MigrationStatusCommand, + MigrationRunCommand, + MigrationApplyCommand, + MigrationTestCommand, + MigrationRollbackCommand, + MigrationRollbackToCommand, + BackupCreateCommand, + BackupRestoreCommand, + BackupListCommand, + BackupDeleteCommand, + MigrationTestSequenceCommand, + ); + } + + return { + module: DbMigrationModule, + imports: [ + TypeOrmModule.forFeature([ + MigrationHistory, + MigrationLock, + BackupMetadata, + ]), + ], + providers, + exports: [ + MigrationService, + RollbackService, + ValidatorService, + BackupService, + TestingService, + ], + }; + } +} + +// 6. Example Usage (in app.module.ts) +/* +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { DbMigrationModule } from './db-migration/db-migration.module'; + +@Module({ + imports: [ + TypeOrmModule.forRoot({ + type: 'postgres', + host: 'localhost', + port: 5432, + username: 'postgres', + password: 'postgres', + database: 'my_database', + entities: [__dirname + '/**/*.entity{.ts,.js}'], + synchronize: false, // Important: set to false when using migrations + }), + DbMigrationModule.register({ + migrationDir: './migrations', + backupDir: './backups', + enableCli: true, + }), + ], +}) +export class AppModule {} +*/ + +// 7. Example Migration (in migrations/1677589200000-create-users-table.ts) +/* +import { QueryRunner } from 'typeorm'; +import { Migration } from '../src/db-migration/interfaces/migration.interface'; + +const migration: Migration = { + id: '1677589200000-create-users-table', + name: 'Create Users Table', + description: 'Creates the users table with basic fields', + timestamp: 1677589200000, + + // Apply migration + async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + email VARCHAR(255) NOT NULL UNIQUE, + password VARCHAR(255) NOT NULL, + full_name VARCHAR(255) NOT NULL, + is_active BOOLEAN DEFAULT true, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + `); + + await queryRunner.query(` + CREATE INDEX idx_users_email ON users(email) + `); + }, + + // Rollback migration + async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX IF EXISTS idx_users_email`); + await queryRunner.query(`DROP TABLE IF EXISTS users`); + }, + + // Validate before applying (optional) + async validateBefore(queryRunner: QueryRunner): Promise { + // Check if users table doesn't exist yet + const result = await queryRunner.query(` + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'users' + ) as exists + `); + + return !result[0].exists; + }, + + // Validate after applying (optional) + async validateAfter(queryRunner: QueryRunner): Promise { + // Check if users table exists and has the correct columns + const result = await queryRunner.query(` + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'users' + `); + + const columns = result.map(r => r.column_name); + const requiredColumns = ['id', 'email', 'password', 'full_name', 'is_active', 'created_at', 'updated_at']; + + return requiredColumns.every(col => columns.includes(col)); + }, + + // Test migration (optional) + async test(queryRunner: QueryRunner): Promise { + // Test inserting and querying data + try { + // Insert test user + await queryRunner.query(` + INSERT INTO users (email, password, full_name) + VALUES ('test@example.com', 'password123', 'Test User') + `); + + // Query the inserted user + const result = await queryRunner.query(` + SELECT * FROM users WHERE email = 'test@example.com' + `); + + return result.length === 1 && result[0].full_name === 'Test User'; + } catch (error) { + return false; + } + }, +}; + +export default migration; +*/ diff --git a/backend/src/db-migration/entities/backup-metadata.entity.ts b/backend/src/db-migration/entities/backup-metadata.entity.ts new file mode 100644 index 00000000..161d1e23 --- /dev/null +++ b/backend/src/db-migration/entities/backup-metadata.entity.ts @@ -0,0 +1,35 @@ +// src/db-migration/entities/backup-metadata.entity.ts +import { Entity, Column, PrimaryGeneratedColumn, CreateDateColumn, Index } from 'typeorm'; + +@Entity('backup_metadata') +export class BackupMetadata { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column() + @Index() + backupId: string; + + @Column() + filename: string; + + @Column() + location: string; + + @Column('bigint') + size: number; + + @Column({ nullable: true }) + @Index() + migrationId?: string; + + @Column({ default: true }) + available: boolean; + + @Column('json', { nullable: true }) + metadata?: any; + + @CreateDateColumn() + @Index() + createdAt: Date; +} \ No newline at end of file diff --git a/backend/src/db-migration/entities/migration-history.entity.ts b/backend/src/db-migration/entities/migration-history.entity.ts new file mode 100644 index 00000000..bb3a9981 --- /dev/null +++ b/backend/src/db-migration/entities/migration-history.entity.ts @@ -0,0 +1,49 @@ +// src/db-migration/entities/migration-history.entity.ts +import { Entity, Column, PrimaryGeneratedColumn, CreateDateColumn, UpdateDateColumn, Index } from 'typeorm'; + +@Entity('migration_history') +export class MigrationHistory { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column() + @Index() + migrationId: string; + + @Column() + name: string; + + @Column({ nullable: true }) + description?: string; + + @Column('json', { nullable: true }) + dependencies?: string[]; + + @Column() + @Index() + appliedAt: Date; + + @Column({ default: true }) + success: boolean; + + @Column({ nullable: true }) + error?: string; + + @Column({ default: false }) + rolledBack: boolean; + + @Column({ nullable: true }) + rolledBackAt?: Date; + + @Column({ nullable: true }) + backupId?: string; + + @Column('json', { nullable: true }) + metadata?: any; + + @CreateDateColumn() + createdAt: Date; + + @UpdateDateColumn() + updatedAt: Date; +} diff --git a/backend/src/db-migration/entities/migration-lock.entity.ts b/backend/src/db-migration/entities/migration-lock.entity.ts new file mode 100644 index 00000000..a6101c33 --- /dev/null +++ b/backend/src/db-migration/entities/migration-lock.entity.ts @@ -0,0 +1,26 @@ +// src/db-migration/entities/migration-lock.entity.ts +import { Entity, Column, PrimaryColumn, UpdateDateColumn } from 'typeorm'; + +@Entity('migration_lock') +export class MigrationLock { + @PrimaryColumn() + id: string; + + @Column() + lockedBy: string; + + @Column() + lockedAt: Date; + + @Column({ nullable: true }) + expiresAt?: Date; + + @Column({ default: false }) + isExecuting: boolean; + + @Column({ nullable: true }) + currentMigrationId?: string; + + @UpdateDateColumn() + updatedAt: Date; +} diff --git a/backend/src/db-migration/interfaces/backup.interface.ts b/backend/src/db-migration/interfaces/backup.interface.ts new file mode 100644 index 00000000..c3d72452 --- /dev/null +++ b/backend/src/db-migration/interfaces/backup.interface.ts @@ -0,0 +1,87 @@ +// src/db-migration/interfaces/backup.interface.ts +export interface BackupProvider { + /** + * Name of the backup provider + */ + name: string; + + /** + * Create a backup of the database + */ + createBackup(connectionOptions: any, metadata: any): Promise; + + /** + * Restore a database from a backup + */ + restoreBackup(backupId: string, connectionOptions: any): Promise; + + /** + * List available backups + */ + listBackups(filter?: any): Promise; + + /** + * Delete a backup + */ + deleteBackup(backupId: string): Promise; + } + + export interface BackupResult { + /** + * Unique identifier for the backup + */ + backupId: string; + + /** + * Success status + */ + success: boolean; + + /** + * Timestamp of the backup + */ + timestamp: number; + + /** + * Size of the backup in bytes + */ + size?: number; + + /** + * Location where the backup is stored + */ + location?: string; + + /** + * Error message if backup failed + */ + error?: string; + } + + export interface BackupInfo { + /** + * Unique identifier for the backup + */ + backupId: string; + + /** + * Timestamp of the backup + */ + timestamp: number; + + /** + * Size of the backup in bytes + */ + size: number; + + /** + * Migration ID if backup was before a migration + */ + migrationId?: string; + + /** + * Additional metadata + */ + metadata?: any; + } + \ No newline at end of file diff --git a/backend/src/db-migration/interfaces/migration.interface.ts b/backend/src/db-migration/interfaces/migration.interface.ts new file mode 100644 index 00000000..c6b1852b --- /dev/null +++ b/backend/src/db-migration/interfaces/migration.interface.ts @@ -0,0 +1,53 @@ +// src/db-migration/interfaces/migration.interface.ts +export interface Migration { + /** + * Unique identifier for the migration + */ + id: string; + + /** + * Human-readable name for the migration + */ + name: string; + + /** + * Optional description of what the migration does + */ + description?: string; + + /** + * Timestamp when the migration was created + */ + timestamp: number; + + /** + * List of migration IDs that must be applied before this one + */ + dependencies?: string[]; + + /** + * Function to apply the migration + */ + up: (queryRunner: any) => Promise; + + /** + * Function to revert the migration + */ + down: (queryRunner: any) => Promise; + + /** + * Optional pre-validation function + */ + validateBefore?: (queryRunner: any) => Promise; + + /** + * Optional post-validation function + */ + validateAfter?: (queryRunner: any) => Promise; + + /** + * Optional function to test the migration + */ + test?: (queryRunner: any) => Promise; + } + \ No newline at end of file diff --git a/backend/src/db-migration/interfaces/validator.interface.ts b/backend/src/db-migration/interfaces/validator.interface.ts new file mode 100644 index 00000000..33221478 --- /dev/null +++ b/backend/src/db-migration/interfaces/validator.interface.ts @@ -0,0 +1,29 @@ +// src/db-migration/interfaces/validator.interface.ts +export interface DataValidator { + /** + * Name of the validator + */ + name: string; + + /** + * Validates data against a schema or set of rules + */ + validate(data: any, schema: any): Promise; + } + + export interface ValidationResult { + /** + * Whether validation passed + */ + valid: boolean; + + /** + * Error messages if validation failed + */ + errors?: string[]; + + /** + * Warnings that don't cause validation to fail + */ + warnings?: string[]; + } \ No newline at end of file diff --git a/backend/src/db-migration/services/backup.service.ts b/backend/src/db-migration/services/backup.service.ts new file mode 100644 index 00000000..74db4e07 --- /dev/null +++ b/backend/src/db-migration/services/backup.service.ts @@ -0,0 +1,512 @@ +// src/db-migration/services/backup.service.ts +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository, Connection } from 'typeorm'; +import { BackupMetadata } from '../entities/backup-metadata.entity'; +import { BackupProvider, BackupResult, BackupInfo } from '../interfaces/backup.interface'; +import * as path from 'path'; +import * as fs from 'fs'; +import { exec } from 'child_process'; +import { promisify } from 'util'; +import { v4 as uuidv4 } from 'uuid'; + +const execPromise = promisify(exec); + +@Injectable() +export class BackupService { + private readonly logger = new Logger(BackupService.name); + private readonly providers: Map = new Map(); + private defaultProvider: string | null = null; + private backupDir: string = './backups'; + + constructor( + @InjectRepository(BackupMetadata) + private backupMetadataRepository: Repository, + private connection: Connection, + ) { + // Register built-in filesystem backup provider + this.registerProvider(new FilesystemBackupProvider(this.backupDir)); + this.setDefaultProvider('filesystem'); + } + + /** + * Set the backup directory + */ + setBackupDirectory(dir: string): void { + this.backupDir = dir; + // Update filesystem provider + const provider = this.providers.get('filesystem'); + if (provider && provider instanceof FilesystemBackupProvider) { + provider.setBackupDirectory(dir); + } + } + + /** + * Register a backup provider + */ + registerProvider(provider: BackupProvider): void { + this.providers.set(provider.name, provider); + this.logger.log(`Registered backup provider: ${provider.name}`); + + // Set as default if no default provider + if (!this.defaultProvider) { + this.defaultProvider = provider.name; + } + } + + /** + * Set the default backup provider + */ + setDefaultProvider(providerName: string): void { + if (!this.providers.has(providerName)) { + throw new Error(`Backup provider ${providerName} not registered`); + } + + this.defaultProvider = providerName; + this.logger.log(`Set default backup provider to ${providerName}`); + } + + /** + * Get a provider by name + */ + getProvider(name?: string): BackupProvider { + const providerName = name || this.defaultProvider; + + if (!providerName) { + throw new Error('No backup provider specified and no default provider set'); + } + + const provider = this.providers.get(providerName); + + if (!provider) { + throw new Error(`Backup provider ${providerName} not found`); + } + + return provider; + } + + /** + * Create a database backup + */ + async createBackup(options: { + reason?: string; + migrationId?: string; + provider?: string; + metadata?: any; + } = {}): Promise { + try { + const provider = this.getProvider(options.provider); + + // Get connection options + const connectionOptions = this.connection.options; + + // Create backup + const result = await provider.createBackup(connectionOptions, { + reason: options.reason || 'manual', + migrationId: options.migrationId, + timestamp: new Date().toISOString(), + ...options.metadata, + }); + + if (result.success) { + // Save backup metadata + await this.backupMetadataRepository.save({ + backupId: result.backupId, + filename: path.basename(result.location || ''), + location: result.location || '', + size: result.size || 0, + migrationId: options.migrationId, + metadata: { + reason: options.reason || 'manual', + provider: provider.name, + timestamp: new Date().toISOString(), + ...options.metadata, + }, + }); + } + + return result; + } catch (error) { + this.logger.error(`Error creating backup: ${error.message}`, error.stack); + + return { + backupId: uuidv4(), + success: false, + timestamp: Date.now(), + error: error.message, + }; + } + } + + /** + * Restore a database from backup + */ + async restoreBackup(backupId: string, options: { + provider?: string; + } = {}): Promise { + try { + // Get backup metadata + const backupMetadata = await this.backupMetadataRepository.findOne({ + where: { backupId }, + }); + + if (!backupMetadata) { + throw new Error(`Backup with ID ${backupId} not found`); + } + + // Get provider (use the one that created the backup or specified provider) + const providerName = options.provider || + (backupMetadata.metadata?.provider || this.defaultProvider); + + const provider = this.getProvider(providerName); + + // Get connection options + const connectionOptions = this.connection.options; + + // Restore backup + const result = await provider.restoreBackup(backupId, connectionOptions); + + if (result) { + this.logger.log(`Successfully restored backup ${backupId}`); + } else { + this.logger.error(`Failed to restore backup ${backupId}`); + } + + return result; + } catch (error) { + this.logger.error(`Error restoring backup: ${error.message}`, error.stack); + return false; + } + } + + /** + * List available backups + */ + async listBackups(options: { + migrationId?: string; + provider?: string; + limit?: number; + offset?: number; + } = {}): Promise { + try { + const query = this.backupMetadataRepository.createQueryBuilder('backup') + .where('backup.available = :available', { available: true }); + + if (options.migrationId) { + query.andWhere('backup.migrationId = :migrationId', { + migrationId: options.migrationId + }); + } + + if (options.provider) { + query.andWhere("backup.metadata->>'provider' = :provider", { + provider: options.provider + }); + } + + if (options.limit) { + query.limit(options.limit); + } + + if (options.offset) { + query.offset(options.offset); + } + + query.orderBy('backup.createdAt', 'DESC'); + + const backups = await query.getMany(); + + return backups.map(backup => ({ + backupId: backup.backupId, + timestamp: backup.createdAt.getTime(), + size: backup.size, + migrationId: backup.migrationId, + metadata: backup.metadata, + })); + } catch (error) { + this.logger.error(`Error listing backups: ${error.message}`, error.stack); + return []; + } + } + + /** + * Delete a backup + */ + async deleteBackup(backupId: string, options: { + provider?: string; + } = {}): Promise { + try { + // Get backup metadata + const backupMetadata = await this.backupMetadataRepository.findOne({ + where: { backupId }, + }); + + if (!backupMetadata) { + throw new Error(`Backup with ID ${backupId} not found`); + } + + // Get provider (use the one that created the backup or specified provider) + const providerName = options.provider || + (backupMetadata.metadata?.provider || this.defaultProvider); + + const provider = this.getProvider(providerName); + + // Delete backup + const result = await provider.deleteBackup(backupId); + + if (result) { + // Update backup metadata + backupMetadata.available = false; + await this.backupMetadataRepository.save(backupMetadata); + + this.logger.log(`Successfully deleted backup ${backupId}`); + } else { + this.logger.error(`Failed to delete backup ${backupId}`); + } + + return result; + } catch (error) { + this.logger.error(`Error deleting backup: ${error.message}`, error.stack); + return false; + } + } +} + +/** + * Filesystem backup provider implementation + */ +class FilesystemBackupProvider implements BackupProvider { + name = 'filesystem'; + private backupDir: string; + + constructor(backupDir: string) { + this.backupDir = backupDir; + this.ensureBackupDirectory(); + } + + setBackupDirectory(dir: string): void { + this.backupDir = dir; + this.ensureBackupDirectory(); + } + + private ensureBackupDirectory(): void { + if (!fs.existsSync(this.backupDir)) { + fs.mkdirSync(this.backupDir, { recursive: true }); + } + } + + async createBackup(connectionOptions: any, metadata: any): Promise { + try { + // Generate backup ID and filename + const backupId = uuidv4(); + const timestamp = new Date().toISOString().replace(/[:\.]/g, '-'); + const filename = `backup-${timestamp}-${backupId}.sql`; + const backupPath = path.join(this.backupDir, filename); + + // Determine database type and run appropriate backup command + let command: string; + + switch (connectionOptions.type) { + case 'postgres': + command = this.buildPostgresBackupCommand(connectionOptions, backupPath); + break; + case 'mysql': + case 'mariadb': + command = this.buildMysqlBackupCommand(connectionOptions, backupPath); + break; + default: + throw new Error(`Unsupported database type: ${connectionOptions.type}`); + } + + // Execute backup command + await execPromise(command); + + // Get backup file size + const stats = fs.statSync(backupPath); + + return { + backupId, + success: true, + timestamp: Date.now(), + size: stats.size, + location: backupPath, + }; + } catch (error) { + return { + backupId: uuidv4(), + success: false, + timestamp: Date.now(), + error: error.message, + }; + } + } + + async restoreBackup(backupId: string, connectionOptions: any): Promise { + try { + // Find backup file by ID + const files = await fs.promises.readdir(this.backupDir); + const backupFile = files.find(f => f.includes(backupId)); + + if (!backupFile) { + throw new Error(`Backup file for ID ${backupId} not found`); + } + + const backupPath = path.join(this.backupDir, backupFile); + + // Determine database type and run appropriate restore command + let command: string; + + switch (connectionOptions.type) { + case 'postgres': + command = this.buildPostgresRestoreCommand(connectionOptions, backupPath); + break; + case 'mysql': + case 'mariadb': + command = this.buildMysqlRestoreCommand(connectionOptions, backupPath); + break; + default: + throw new Error(`Unsupported database type: ${connectionOptions.type}`); + } + + // Execute restore command + await execPromise(command); + + return true; + } catch (error) { + console.error('Restore error:', error); + return false; + } + } + + async listBackups(filter?: any): Promise { + try { + const files = await fs.promises.readdir(this.backupDir); + const backupInfos: BackupInfo[] = []; + + for (const file of files) { + if (file.startsWith('backup-') && file.endsWith('.sql')) { + const backupIdMatch = file.match(/backup-.*-([\w-]+)\.sql$/); + const backupId = backupIdMatch ? backupIdMatch[1] : file; + + const stats = fs.statSync(path.join(this.backupDir, file)); + + backupInfos.push({ + backupId, + timestamp: stats.mtime.getTime(), + size: stats.size, + }); + } + } + + return backupInfos; + } catch (error) { + console.error('List backups error:', error); + return []; + } + } + + async deleteBackup(backupId: string): Promise { + try { + // Find backup file by ID + const files = await fs.promises.readdir(this.backupDir); + const backupFile = files.find(f => f.includes(backupId)); + + if (!backupFile) { + throw new Error(`Backup file for ID ${backupId} not found`); + } + + const backupPath = path.join(this.backupDir, backupFile); + + // Delete file + await fs.promises.unlink(backupPath); + + return true; + } catch (error) { + console.error('Delete backup error:', error); + return false; + } + } + + private buildPostgresBackupCommand(connectionOptions: any, backupPath: string): string { + const { + host = 'localhost', + port = 5432, + username, + password, + database, + } = connectionOptions; + + // Build pg_dump command + let command = `PGPASSWORD='${password}' pg_dump`; + command += ` -h ${host}`; + command += ` -p ${port}`; + command += ` -U ${username}`; + command += ` -F p`; // Plain text format + command += ` -f "${backupPath}"`; + command += ` ${database}`; + + return command; + } + + private buildPostgresRestoreCommand(connectionOptions: any, backupPath: string): string { + const { + host = 'localhost', + port = 5432, + username, + password, + database, + } = connectionOptions; + + // Build psql command + let command = `PGPASSWORD='${password}' psql`; + command += ` -h ${host}`; + command += ` -p ${port}`; + command += ` -U ${username}`; + command += ` -d ${database}`; + command += ` -f "${backupPath}"`; + + return command; + } + + private buildMysqlBackupCommand(connectionOptions: any, backupPath: string): string { + const { + host = 'localhost', + port = 3306, + username, + password, + database, + } = connectionOptions; + + // Build mysqldump command + let command = `mysqldump`; + command += ` -h${host}`; + command += ` -P${port}`; + command += ` -u${username}`; + command += ` -p${password}`; + command += ` --result-file="${backupPath}"`; + command += ` ${database}`; + + return command; + } + + private buildMysqlRestoreCommand(connectionOptions: any, backupPath: string): string { + const { + host = 'localhost', + port = 3306, + username, + password, + database, + } = connectionOptions; + + // Build mysql command + let command = `mysql`; + command += ` -h${host}`; + command += ` -P${port}`; + command += ` -u${username}`; + command += ` -p${password}`; + command += ` ${database}`; + command += ` < "${backupPath}"`; + + return command; + } +} \ No newline at end of file diff --git a/backend/src/db-migration/services/migration.service.ts b/backend/src/db-migration/services/migration.service.ts new file mode 100644 index 00000000..d5138ab3 --- /dev/null +++ b/backend/src/db-migration/services/migration.service.ts @@ -0,0 +1,463 @@ +// src/db-migration/services/migration.service.ts +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository, Connection, QueryRunner } from 'typeorm'; +import { v4 as uuidv4 } from 'uuid'; +import { MigrationHistory } from '../entities/migration-history.entity'; +import { MigrationLock } from '../entities/migration-lock.entity'; +import { Migration } from '../interfaces/migration.interface'; +import { BackupService } from './backup.service'; +import { ValidatorService } from './validator.service'; +import * as path from 'path'; +import * as fs from 'fs'; + +@Injectable() +export class MigrationService { + private readonly logger = new Logger(MigrationService.name); + private readonly migrations: Map = new Map(); + private readonly lockId = 'migration_lock'; + private readonly maxLockTime = 30 * 60 * 1000; // 30 minutes + + constructor( + @InjectRepository(MigrationHistory) + private migrationHistoryRepository: Repository, + @InjectRepository(MigrationLock) + private migrationLockRepository: Repository, + private connection: Connection, + private backupService: BackupService, + private validatorService: ValidatorService, + ) {} + + /** + * Initialize the migration service + */ + async init(migrationDir: string): Promise { + try { + // Load all migrations from the specified directory + await this.loadMigrations(migrationDir); + + // Initialize migration lock + await this.initMigrationLock(); + + this.logger.log(`Migration service initialized with ${this.migrations.size} migrations`); + } catch (error) { + this.logger.error(`Failed to initialize migration service: ${error.message}`, error.stack); + throw error; + } + } + + /** + * Load migrations from the specified directory + */ + private async loadMigrations(migrationDir: string): Promise { + const migrationFiles = await fs.promises.readdir(migrationDir); + + for (const file of migrationFiles) { + if (file.endsWith('.js') || file.endsWith('.ts')) { + try { + const migrationModule = await import(path.join(migrationDir, file)); + const migration: Migration = migrationModule.default; + + if (this.isValidMigration(migration)) { + this.migrations.set(migration.id, migration); + this.logger.verbose(`Loaded migration: ${migration.name} (${migration.id})`); + } else { + this.logger.warn(`Invalid migration in file: ${file}`); + } + } catch (error) { + this.logger.error(`Failed to load migration file ${file}: ${error.message}`); + } + } + } + } + + /** + * Check if a migration object is valid + */ + private isValidMigration(migration: any): boolean { + return ( + migration && + typeof migration.id === 'string' && + typeof migration.name === 'string' && + typeof migration.timestamp === 'number' && + typeof migration.up === 'function' && + typeof migration.down === 'function' + ); + } + + /** + * Initialize migration lock + */ + private async initMigrationLock(): Promise { + const existingLock = await this.migrationLockRepository.findOne(this.lockId); + + if (!existingLock) { + await this.migrationLockRepository.save({ + id: this.lockId, + lockedBy: 'none', + lockedAt: new Date(0), + isExecuting: false, + }); + } + } + + /** + * Get pending migrations + */ + async getPendingMigrations(): Promise { + const appliedMigrations = await this.migrationHistoryRepository.find({ + where: { success: true, rolledBack: false }, + select: ['migrationId'], + }); + + const appliedIds = new Set(appliedMigrations.map(m => m.migrationId)); + + return Array.from(this.migrations.values()) + .filter(migration => !appliedIds.has(migration.id)) + .sort((a, b) => a.timestamp - b.timestamp); + } + + /** + * Get applied migrations + */ + async getAppliedMigrations(): Promise { + return this.migrationHistoryRepository.find({ + where: { success: true, rolledBack: false }, + order: { appliedAt: 'DESC' }, + }); + } + + /** + * Apply pending migrations + */ + async applyPendingMigrations(options: { + createBackup?: boolean; + validate?: boolean; + transaction?: boolean; + } = {}): Promise { + const { + createBackup = true, + validate = true, + transaction = true, + } = options; + + // Acquire lock + const lock = await this.acquireLock(); + if (!lock) { + throw new Error('Failed to acquire migration lock. Another migration might be in progress.'); + } + + try { + const pendingMigrations = await this.getPendingMigrations(); + + if (pendingMigrations.length === 0) { + this.logger.log('No pending migrations to apply'); + return []; + } + + this.logger.log(`Applying ${pendingMigrations.length} pending migrations`); + + const results: MigrationHistory[] = []; + + for (const migration of pendingMigrations) { + let backupId: string | undefined; + + try { + // Update lock with current migration + await this.updateLock(migration.id); + + // Create backup if enabled + if (createBackup) { + const backupResult = await this.backupService.createBackup({ + reason: 'pre-migration', + migrationId: migration.id, + metadata: { + migrationName: migration.name, + timestamp: new Date().toISOString(), + }, + }); + + if (backupResult.success) { + backupId = backupResult.backupId; + this.logger.log(`Created backup ${backupId} before migration ${migration.id}`); + } else { + throw new Error(`Failed to create backup: ${backupResult.error}`); + } + } + + // Create query runner and start transaction if enabled + const queryRunner = this.connection.createQueryRunner(); + await queryRunner.connect(); + + if (transaction) { + await queryRunner.startTransaction(); + } + + try { + // Run pre-validation if enabled and defined + if (validate && migration.validateBefore) { + const validationResult = await migration.validateBefore(queryRunner); + if (!validationResult) { + throw new Error(`Pre-migration validation failed for migration ${migration.id}`); + } + } + + // Apply migration + await migration.up(queryRunner); + + // Run post-validation if enabled and defined + if (validate && migration.validateAfter) { + const validationResult = await migration.validateAfter(queryRunner); + if (!validationResult) { + throw new Error(`Post-migration validation failed for migration ${migration.id}`); + } + } + + // Commit transaction if enabled + if (transaction) { + await queryRunner.commitTransaction(); + } + + // Record successful migration + const historyEntry = await this.recordMigration(migration, true, undefined, backupId); + results.push(historyEntry); + + this.logger.log(`Successfully applied migration: ${migration.name} (${migration.id})`); + } catch (error) { + // Rollback transaction if enabled + if (transaction) { + await queryRunner.rollbackTransaction(); + } + + // Record failed migration + const historyEntry = await this.recordMigration( + migration, + false, + error.message, + backupId + ); + results.push(historyEntry); + + this.logger.error( + `Failed to apply migration ${migration.id}: ${error.message}`, + error.stack + ); + + // Stop processing further migrations + break; + } finally { + // Release query runner + await queryRunner.release(); + } + } catch (error) { + this.logger.error( + `Error while handling migration ${migration.id}: ${error.message}`, + error.stack + ); + break; + } + } + + return results; + } finally { + // Release lock + await this.releaseLock(); + } + } + + /** + * Apply a specific migration + */ + async applyMigration(migrationId: string, options: { + createBackup?: boolean; + validate?: boolean; + transaction?: boolean; + } = {}): Promise { + const migration = this.migrations.get(migrationId); + + if (!migration) { + throw new Error(`Migration with ID ${migrationId} not found`); + } + + // Check if migration has already been applied + const existingMigration = await this.migrationHistoryRepository.findOne({ + where: { migrationId, success: true, rolledBack: false }, + }); + + if (existingMigration) { + throw new Error(`Migration ${migrationId} has already been applied`); + } + + // Acquire lock + const lock = await this.acquireLock(); + if (!lock) { + throw new Error('Failed to acquire migration lock. Another migration might be in progress.'); + } + + try { + // Update lock with current migration + await this.updateLock(migration.id); + + let backupId: string | undefined; + + // Create backup if enabled + if (options.createBackup !== false) { + const backupResult = await this.backupService.createBackup({ + reason: 'pre-migration', + migrationId: migration.id, + metadata: { + migrationName: migration.name, + timestamp: new Date().toISOString(), + }, + }); + + if (backupResult.success) { + backupId = backupResult.backupId; + this.logger.log(`Created backup ${backupId} before migration ${migration.id}`); + } else { + throw new Error(`Failed to create backup: ${backupResult.error}`); + } + } + + // Create query runner and start transaction if enabled + const queryRunner = this.connection.createQueryRunner(); + await queryRunner.connect(); + + if (options.transaction !== false) { + await queryRunner.startTransaction(); + } + + try { + // Run pre-validation if enabled and defined + if (options.validate !== false && migration.validateBefore) { + const validationResult = await migration.validateBefore(queryRunner); + if (!validationResult) { + throw new Error(`Pre-migration validation failed for migration ${migration.id}`); + } + } + + // Apply migration + await migration.up(queryRunner); + + // Run post-validation if enabled and defined + if (options.validate !== false && migration.validateAfter) { + const validationResult = await migration.validateAfter(queryRunner); + if (!validationResult) { + throw new Error(`Post-migration validation failed for migration ${migration.id}`); + } + } + + // Commit transaction if enabled + if (options.transaction !== false) { + await queryRunner.commitTransaction(); + } + + // Record successful migration + const historyEntry = await this.recordMigration(migration, true, undefined, backupId); + + this.logger.log(`Successfully applied migration: ${migration.name} (${migration.id})`); + + return historyEntry; + } catch (error) { + // Rollback transaction if enabled + if (options.transaction !== false) { + await queryRunner.rollbackTransaction(); + } + + // Record failed migration + const historyEntry = await this.recordMigration( + migration, + false, + error.message, + backupId + ); + + this.logger.error( + `Failed to apply migration ${migration.id}: ${error.message}`, + error.stack + ); + + return historyEntry; + } finally { + // Release query runner + await queryRunner.release(); + } + } finally { + // Release lock + await this.releaseLock(); + } + } + + /** + * Record migration in history + */ + private async recordMigration( + migration: Migration, + success: boolean, + error?: string, + backupId?: string, + ): Promise { + const historyEntry = this.migrationHistoryRepository.create({ + migrationId: migration.id, + name: migration.name, + description: migration.description, + dependencies: migration.dependencies, + appliedAt: new Date(), + success, + error, + backupId, + metadata: { + timestamp: migration.timestamp, + }, + }); + + return this.migrationHistoryRepository.save(historyEntry); + } + + /** + * Acquire migration lock + */ + private async acquireLock(): Promise { + const now = new Date(); + const instanceId = uuidv4(); + + // Try to acquire lock or take it if expired + const result = await this.migrationLockRepository.createQueryBuilder() + .update(MigrationLock) + .set({ + lockedBy: instanceId, + lockedAt: now, + expiresAt: new Date(now.getTime() + this.maxLockTime), + isExecuting: true, + }) + .where('id = :lockId', { lockId: this.lockId }) + .andWhere('(lockedBy = :none OR expiresAt < :now)', { none: 'none', now }) + .execute(); + + return result.affected > 0; + } + + /** + * Update lock with current migration ID + */ + private async updateLock(migrationId: string): Promise { + await this.migrationLockRepository.update( + { id: this.lockId }, + { currentMigrationId: migrationId } + ); + } + + /** + * Release migration lock + */ + private async releaseLock(): Promise { + await this.migrationLockRepository.update( + { id: this.lockId }, + { + isExecuting: false, + currentMigrationId: null, + lockedBy: 'none', + } + ); + } +} diff --git a/backend/src/db-migration/services/rollback.service.ts b/backend/src/db-migration/services/rollback.service.ts new file mode 100644 index 00000000..69514e85 --- /dev/null +++ b/backend/src/db-migration/services/rollback.service.ts @@ -0,0 +1,181 @@ +// src/db-migration/services/rollback.service.ts +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository, Connection } from 'typeorm'; +import { MigrationHistory } from '../entities/migration-history.entity'; +import { Migration } from '../interfaces/migration.interface'; +import { MigrationService } from './migration.service'; +import { BackupService } from './backup.service'; + +@Injectable() +export class RollbackService { + private readonly logger = new Logger(RollbackService.name); + + constructor( + @InjectRepository(MigrationHistory) + private migrationHistoryRepository: Repository, + private connection: Connection, + private migrationService: MigrationService, + private backupService: BackupService, + ) {} + + /** + * Rollback the last migration + */ + async rollbackLastMigration(options: { + transaction?: boolean; + useBackup?: boolean; + } = {}): Promise { + const lastMigration = await this.migrationHistoryRepository.findOne({ + where: { success: true, rolledBack: false }, + order: { appliedAt: 'DESC' }, + }); + + if (!lastMigration) { + this.logger.log('No migrations to rollback'); + return null; + } + + return this.rollbackMigration(lastMigration.migrationId, options); + } + + /** + * Rollback a specific migration + */ + async rollbackMigration(migrationId: string, options: { + transaction?: boolean; + useBackup?: boolean; + } = {}): Promise { + const { + transaction = true, + useBackup = false, + } = options; + + // Find migration history entry + const migrationHistory = await this.migrationHistoryRepository.findOne({ + where: { migrationId, success: true, rolledBack: false }, + }); + + if (!migrationHistory) { + throw new Error(`Migration ${migrationId} has not been applied or has already been rolled back`); + } + + // Try to use backup if specified + if (useBackup && migrationHistory.backupId) { + this.logger.log(`Rolling back migration ${migrationId} using backup ${migrationHistory.backupId}`); + + const restoreResult = await this.backupService.restoreBackup(migrationHistory.backupId); + + if (restoreResult) { + // Mark as rolled back + migrationHistory.rolledBack = true; + migrationHistory.rolledBackAt = new Date(); + await this.migrationHistoryRepository.save(migrationHistory); + + this.logger.log(`Successfully rolled back migration ${migrationId} using backup`); + return migrationHistory; + } else { + this.logger.warn(`Failed to restore backup ${migrationHistory.backupId}, will try manual rollback`); + } + } + + // Get migration definition + const migrations = await this.migrationService['migrations']; + const migration = migrations.get(migrationId); + + if (!migration) { + throw new Error(`Migration definition for ${migrationId} not found`); + } + + // Create query runner and start transaction if enabled + const queryRunner = this.connection.createQueryRunner(); + await queryRunner.connect(); + + if (transaction) { + await queryRunner.startTransaction(); + } + + try { + // Execute rollback + await migration.down(queryRunner); + + // Commit transaction if enabled + if (transaction) { + await queryRunner.commitTransaction(); + } + + // Mark as rolled back + migrationHistory.rolledBack = true; + migrationHistory.rolledBackAt = new Date(); + await this.migrationHistoryRepository.save(migrationHistory); + + this.logger.log(`Successfully rolled back migration ${migrationId}`); + return migrationHistory; + } catch (error) { + // Rollback transaction if enabled + if (transaction) { + await queryRunner.rollbackTransaction(); + } + + this.logger.error( + `Failed to roll back migration ${migrationId}: ${error.message}`, + error.stack + ); + + throw error; + } finally { + // Release query runner + await queryRunner.release(); + } + } + + /** + * Rollback all migrations to a specific point + */ + async rollbackToMigration(targetMigrationId: string, options: { + transaction?: boolean; + useBackup?: boolean; + } = {}): Promise { + // Find all migrations applied after the target + const migrations = await this.migrationHistoryRepository.find({ + where: { success: true, rolledBack: false }, + order: { appliedAt: 'DESC' }, + }); + + // Find target index + const targetIndex = migrations.findIndex(m => m.migrationId === targetMigrationId); + + if (targetIndex === -1) { + throw new Error(`Target migration ${targetMigrationId} not found or not applied`); + } + + // Get migrations to rollback (all after target) + const migrationsToRollback = migrations.slice(0, targetIndex); + + if (migrationsToRollback.length === 0) { + this.logger.log(`No migrations to rollback to ${targetMigrationId}`); + return []; + } + + this.logger.log(`Rolling back ${migrationsToRollback.length} migrations to ${targetMigrationId}`); + + const results: MigrationHistory[] = []; + + // Rollback each migration in reverse order + for (const migration of migrationsToRollback) { + try { + const result = await this.rollbackMigration(migration.migrationId, options); + if (result) { + results.push(result); + } + } catch (error) { + this.logger.error( + `Rollback process stopped due to error on migration ${migration.migrationId}: ${error.message}` + ); + break; + } + } + + return results; + } +} diff --git a/backend/src/db-migration/services/testing.service.ts b/backend/src/db-migration/services/testing.service.ts new file mode 100644 index 00000000..59a1bdde --- /dev/null +++ b/backend/src/db-migration/services/testing.service.ts @@ -0,0 +1,209 @@ +// src/db-migration/services/testing.service.ts +import { Injectable, Logger } from '@nestjs/common'; +import { Connection } from 'typeorm'; +import { Migration } from '../interfaces/migration.interface'; +import { MigrationService } from './migration.service'; +import { RollbackService } from './rollback.service'; +import { ValidatorService } from './validator.service'; + +@Injectable() +export class TestingService { + private readonly logger = new Logger(TestingService.name); + + constructor( + private connection: Connection, + private migrationService: MigrationService, + private rollbackService: RollbackService, + private validatorService: ValidatorService, + ) {} + + /** + * Test a migration + */ + async testMigration(migrationId: string): Promise<{ + success: boolean; + results: { + up: boolean; + down: boolean; + validation: boolean; + test?: boolean; + }; + errors?: string[]; + }> { + try { + // Get migration + const migrations = await this.migrationService['migrations']; + const migration = migrations.get(migrationId); + + if (!migration) { + throw new Error(`Migration ${migrationId} not found`); + } + + // Create test connection using the same config but with a separate schema + // This is a simplified approach for the example + const testSchema = `test_${migrationId.substring(0, 8)}`; + + // Create a schema for testing (this is PostgreSQL specific) + const queryRunner = this.connection.createQueryRunner(); + await queryRunner.connect(); + + try { + // Create test schema + await queryRunner.query(`CREATE SCHEMA IF NOT EXISTS ${testSchema}`); + + const errors: string[] = []; + const results = { + up: false, + down: false, + validation: false, + test: undefined as boolean | undefined, + }; + + try { + // Test migration up + await this.testMigrationUp(migration, queryRunner, testSchema); + results.up = true; + + // Test validation if available + if (migration.validateAfter) { + const validationResult = await migration.validateAfter(queryRunner); + results.validation = validationResult; + + if (!validationResult) { + errors.push('Post-migration validation failed'); + } + } + + // Run migration-specific tests if available + if (migration.test) { + const testResult = await migration.test(queryRunner); + results.test = testResult; + + if (!testResult) { + errors.push('Migration test failed'); + } + } + + // Test migration down + await this.testMigrationDown(migration, queryRunner); + results.down = true; + } catch (error) { + errors.push(`Test error: ${error.message}`); + this.logger.error(`Migration test error: ${error.message}`, error.stack); + } + + return { + success: errors.length === 0 && results.up && results.down, + results, + errors: errors.length > 0 ? errors : undefined, + }; + } finally { + // Drop test schema + await queryRunner.query(`DROP SCHEMA IF EXISTS ${testSchema} CASCADE`); + + // Release query runner + await queryRunner.release(); + } + } catch (error) { + this.logger.error(`Error testing migration: ${error.message}`, error.stack); + + return { + success: false, + results: { + up: false, + down: false, + validation: false, + }, + errors: [`Error in test setup: ${error.message}`], + }; + } + } + + /** + * Test migration up + */ + private async testMigrationUp( + migration: Migration, + queryRunner: any, + schema: string, + ): Promise { + // Set search path to test schema (PostgreSQL specific) + await queryRunner.query(`SET search_path TO ${schema}`); + + // Run pre-validation if available + if (migration.validateBefore) { + const validationResult = await migration.validateBefore(queryRunner); + if (!validationResult) { + throw new Error('Pre-migration validation failed in test'); + } + } + + // Run migration up + await migration.up(queryRunner); + } + + /** + * Test migration down + */ + private async testMigrationDown( + migration: Migration, + queryRunner: any, + ): Promise { + // Run migration down + await migration.down(queryRunner); + } + + /** + * Test migrations in sequence + */ + async testMigrationSequence(migrationIds: string[]): Promise<{ + success: boolean; + results: Record; + errors?: Record; + }> { + try { + const results: Record = {}; + + const errors: Record = {}; + let hasErrors = false; + + // Test each migration in the sequence + for (const migrationId of migrationIds) { + const migrationResult = await this.testMigration(migrationId); + + results[migrationId] = migrationResult.results; + + if (!migrationResult.success) { + hasErrors = true; + errors[migrationId] = migrationResult.errors || ['Unknown error']; + } + } + + return { + success: !hasErrors, + results, + errors: hasErrors ? errors : undefined, + }; + } catch (error) { + this.logger.error(`Error testing migration sequence: ${error.message}`, error.stack); + + return { + success: false, + results: {}, + errors: { + general: [`Error in test sequence: ${error.message}`], + }, + }; + } + } +} diff --git a/backend/src/db-migration/services/validator.service.ts b/backend/src/db-migration/services/validator.service.ts new file mode 100644 index 00000000..644908d6 --- /dev/null +++ b/backend/src/db-migration/services/validator.service.ts @@ -0,0 +1,248 @@ +// src/db-migration/services/validator.service.ts +import { Injectable, Logger } from '@nestjs/common'; +import { InjectConnection } from '@nestjs/typeorm'; +import { Connection } from 'typeorm'; +import { DataValidator, ValidationResult } from '../interfaces/validator.interface'; + +@Injectable() +export class ValidatorService { + private readonly logger = new Logger(ValidatorService.name); + private readonly validators: Map = new Map(); + + constructor( + @InjectConnection() + private connection: Connection, + ) {} + + /** + * Register a data validator + */ + registerValidator(validator: DataValidator): void { + this.validators.set(validator.name, validator); + this.logger.log(`Registered validator: ${validator.name}`); + } + + /** + * Get a validator by name + */ + getValidator(name: string): DataValidator | undefined { + return this.validators.get(name); + } + + /** + * List all registered validators + */ + listValidators(): DataValidator[] { + return Array.from(this.validators.values()); + } + + /** + * Validate table schema + */ + async validateTableSchema(tableName: string, schema: any): Promise { + try { + const queryRunner = this.connection.createQueryRunner(); + await queryRunner.connect(); + + try { + // Get table information from database + const table = await queryRunner.getTable(tableName); + + if (!table) { + return { + valid: false, + errors: [`Table ${tableName} does not exist`], + }; + } + + const errors: string[] = []; + const warnings: string[] = []; + + // Check columns + for (const columnDef of schema.columns) { + const column = table.findColumnByName(columnDef.name); + + if (!column) { + errors.push(`Column ${columnDef.name} not found in table ${tableName}`); + continue; + } + + // Check column type + if (columnDef.type && !this.isCompatibleType(column.type, columnDef.type)) { + errors.push( + `Column ${columnDef.name} has type ${column.type} but expected ${columnDef.type}` + ); + } + + // Check if nullable + if (columnDef.nullable !== undefined && column.isNullable !== columnDef.nullable) { + errors.push( + `Column ${columnDef.name} has nullable=${column.isNullable} but expected ${columnDef.nullable}` + ); + } + } + + // Check if any columns are in the table but not in the schema + for (const column of table.columns) { + const columnDef = schema.columns.find(c => c.name === column.name); + + if (!columnDef && !column.isGenerated) { + warnings.push(`Column ${column.name} exists in table but not in schema`); + } + } + + return { + valid: errors.length === 0, + errors: errors.length > 0 ? errors : undefined, + warnings: warnings.length > 0 ? warnings : undefined, + }; + } finally { + // Release query runner + await queryRunner.release(); + } + } catch (error) { + this.logger.error(`Error validating table schema: ${error.message}`, error.stack); + + return { + valid: false, + errors: [`Error validating schema: ${error.message}`], + }; + } + } + + /** + * Validate data integrity + */ + async validateDataIntegrity( + tableName: string, + conditions: any[], + ): Promise { + try { + const queryRunner = this.connection.createQueryRunner(); + await queryRunner.connect(); + + try { + const errors: string[] = []; + + // Check each condition + for (const condition of conditions) { + // Build validation query + let query = `SELECT COUNT(*) as count FROM ${tableName} WHERE `; + + if (typeof condition === 'string') { + // Raw SQL condition + query += condition; + } else if (typeof condition === 'object') { + // Object with column-value pairs + const clauses = Object.entries(condition) + .map(([column, value]) => { + if (value === null) { + return `${column} IS NULL`; + } else if (typeof value === 'string') { + return `${column} = '${value.replace(/'/g, "''")}'`; + } else { + return `${column} = ${value}`; + } + }); + + query += clauses.join(' AND '); + } + + // Execute validation query + const result = await queryRunner.query(query); + const count = parseInt(result[0]?.count || '0', 10); + + if (count === 0) { + errors.push(`Data integrity check failed: ${query}`); + } + } + + return { + valid: errors.length === 0, + errors: errors.length > 0 ? errors : undefined, + }; + } finally { + // Release query runner + await queryRunner.release(); + } + } catch (error) { + this.logger.error(`Error validating data integrity: ${error.message}`, error.stack); + + return { + valid: false, + errors: [`Error validating data integrity: ${error.message}`], + }; + } + } + + /** + * Validate custom constraint + */ + async validateCustomConstraint( + constraint: (queryRunner: any) => Promise<{ valid: boolean; message?: string }>, + ): Promise { + try { + const queryRunner = this.connection.createQueryRunner(); + await queryRunner.connect(); + + try { + const result = await constraint(queryRunner); + + return { + valid: result.valid, + errors: result.valid ? undefined : [result.message || 'Custom constraint failed'], + }; + } finally { + // Release query runner + await queryRunner.release(); + } + } catch (error) { + this.logger.error(`Error validating custom constraint: ${error.message}`, error.stack); + + return { + valid: false, + errors: [`Error validating custom constraint: ${error.message}`], + }; + } + } + + /** + * Check if database column types are compatible + */ + private isCompatibleType(actual: string, expected: string): boolean { + // Normalize types to lowercase + const normalizedActual = actual.toLowerCase(); + const normalizedExpected = expected.toLowerCase(); + + // Direct match + if (normalizedActual === normalizedExpected) { + return true; + } + + // Handle type aliases and compatibility + // For example, 'int' is compatible with 'integer' + const compatibilityMap: Record = { + 'int': ['integer', 'smallint', 'bigint'], + 'integer': ['int', 'smallint', 'bigint'], + 'varchar': ['character varying', 'text', 'string'], + 'text': ['varchar', 'character varying', 'string'], + 'float': ['double precision', 'real', 'decimal', 'numeric'], + 'numeric': ['decimal', 'float', 'double precision', 'real'], + 'boolean': ['bool'], + 'timestamp': ['timestamp without time zone', 'datetime'], + 'timestamptz': ['timestamp with time zone'], + }; + + // Check if actual type is compatible with expected type + for (const [baseType, compatibleTypes] of Object.entries(compatibilityMap)) { + if ( + (normalizedExpected === baseType && compatibleTypes.includes(normalizedActual)) || + (normalizedActual === baseType && compatibleTypes.includes(normalizedExpected)) + ) { + return true; + } + } + + return false; + } +}