diff --git a/migrate/README.md b/migrate/README.md new file mode 100644 index 0000000..00b45c7 --- /dev/null +++ b/migrate/README.md @@ -0,0 +1,297 @@ +# migrate + +[![GoDoc Widget]][GoDoc] + +The `migrate` package provides functionality for applying database migrations in your Go applications. It leverages [github.com/rubenv/sql-migrate](https://github.com/rubenv/sql-migrate) under the hood, ensuring a reliable and consistent approach to managing database schema changes. + +## Overview + +The `migrate` package offers two primary approaches for defining your migrations: +- **Embedded SQL Migrations**: Store your migrations as plain SQL files (with separate `.up.sql` and `.down.sql` files) and embed them into your Go binary using Go's built-in embed package. This approach is straightforward and keeps your SQL scripts separate from your application code. +- **Programmatic SQL Migrations**: Define your migrations directly in Go code. This method is more suitable when you require additional customization or more control over your migrations. It lets you write migrations as Go functions, while still leveraging SQL commands. + +## Usage + +The examples below show how to define migrations for creating a "users" table and a "notes" table. + +### Running Embedded SQL Migrations + +You can embed your SQL migration files into your binary with Go's embed package. +The following example (from the [examples/embedded-sql-migrations](./examples/embedded-sql-migrations) directory) demonstrates how to load and execute embedded migrations: + +```go +package main + +import ( + "database/sql" + "embed" + "flag" + "fmt" + stdlog "log" + "os" + + "github.com/acronis/go-appkit/log" + _ "github.com/go-sql-driver/mysql" + _ "github.com/jackc/pgx/v5/stdlib" + _ "github.com/lib/pq" + + "github.com/acronis/go-dbkit" + "github.com/acronis/go-dbkit/migrate" +) + +//go:embed mysql/*.sql +//go:embed postgres/*.sql +var migrationFS embed.FS + +func main() { + if err := runMigrations(); err != nil { + stdlog.Fatal(err) + } +} + +func runMigrations() error { + var migrateDown bool + flag.BoolVar(&migrateDown, "down", false, "migrate down") + var driverName string + flag.StringVar(&driverName, "driver", "", "driver name, supported values: mysql, postgres, pgx") + flag.Parse() + + migrationDirection := migrate.MigrationsDirectionUp + if migrateDown { + migrationDirection = migrate.MigrationsDirectionDown + } + + dialect, migrationDirName, err := parseDialectFromDriver(driverName) + if err != nil { + return fmt.Errorf("parse dialect: %w", err) + } + + dbConn, err := sql.Open(driverName, os.Getenv("DB_DSN")) + if err != nil { + return fmt.Errorf("open database: %w", err) + } + + logger, loggerClose := log.NewLogger(&log.Config{Output: log.OutputStderr, Level: log.LevelInfo}) + defer loggerClose() + + migrationManager, err := migrate.NewMigrationsManager(dbConn, dialect, logger) + if err != nil { + return err + } + migrations, err := migrate.LoadAllEmbedFSMigrations(migrationFS, migrationDirName) + if err != nil { + return fmt.Errorf("make embed fs migrations: %w", err) + } + return migrationManager.Run(migrations, migrationDirection) +} + +func parseDialectFromDriver(driverName string) (dialect dbkit.Dialect, migrationDirName string, err error) { + switch driverName { + case "mysql": + return dbkit.DialectMySQL, "mysql", nil + case "postgres": + return dbkit.DialectPostgres, "postgres", nil + case "pgx": + return dbkit.DialectPgx, "postgres", nil + default: + return "", "", fmt.Errorf("unknown driver name: %s", driverName) + } +} +``` + +### Defining SQL Migrations in Go Files + +For greater control or when you need to include custom logic, you can define your migrations directly in Go. +This approach is demonstrated in the following example from the [examples/go-sql-migrations](examples/go-sql-migrations) directory. +The example includes two migration files that define the creation and deletion of the "users" and "notes" tables. + +**Main Application** + +```go +package main + +import ( + "database/sql" + "flag" + "fmt" + stdlog "log" + "os" + + "github.com/acronis/go-appkit/log" + _ "github.com/go-sql-driver/mysql" + _ "github.com/jackc/pgx/v5/stdlib" + _ "github.com/lib/pq" + + "github.com/acronis/go-dbkit" + "github.com/acronis/go-dbkit/migrate" +) + +func main() { + if err := runMigrations(); err != nil { + stdlog.Fatal(err) + } +} + +func runMigrations() error { + var migrateDown bool + flag.BoolVar(&migrateDown, "down", false, "migrate down") + var driverName string + flag.StringVar(&driverName, "driver", "", "driver name, supported values: mysql, postgres, pgx") + flag.Parse() + + migrationDirection := migrate.MigrationsDirectionUp + if migrateDown { + migrationDirection = migrate.MigrationsDirectionDown + } + + dialect, err := parseDialectFromDriver(driverName) + if err != nil { + return fmt.Errorf("parse dialect: %w", err) + } + + dbConn, err := sql.Open(driverName, os.Getenv("DB_DSN")) + if err != nil { + return fmt.Errorf("open database: %w", err) + } + + logger, loggerClose := log.NewLogger(&log.Config{Output: log.OutputStderr, Level: log.LevelInfo}) + defer loggerClose() + + migrationManager, err := migrate.NewMigrationsManager(dbConn, dialect, logger) + if err != nil { + return err + } + return migrationManager.Run([]migrate.Migration{ + NewMigration0001CreateUsersTable(dialect), + NewMigration0002CreateNotesTable(dialect), + }, migrationDirection) +} + +func parseDialectFromDriver(driverName string) (dbkit.Dialect, error) { + switch driverName { + case "mysql": + return dbkit.DialectMySQL, nil + case "postgres": + return dbkit.DialectPostgres, nil + case "pgx": + return dbkit.DialectPgx, nil + default: + return "", fmt.Errorf("unknown driver name: %s", driverName) + } +} +``` + +**Migration for Creating the Users Table** + +```go +package main + +import ( + "github.com/acronis/go-dbkit" + "github.com/acronis/go-dbkit/migrate" +) + +const migration0001CreateUsersTableUpMySQL = ` +CREATE TABLE users ( + id BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, + name VARCHAR(255) NOT NULL +); +` + +const migration0001CreateUsersTableUpPostgres = ` +CREATE TABLE users ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name VARCHAR(255) NOT NULL +); +` + +const migration0001CreateUsersTableDown = ` +DROP TABLE users; +` + +type Migration0001CreateUsersTable struct { + *migrate.NullMigration +} + +func NewMigration0001CreateUsersTable(dialect dbkit.Dialect) *Migration0001CreateUsersTable { + return &Migration0001CreateUsersTable{&migrate.NullMigration{Dialect: dialect}} +} + +func (m *Migration0001CreateUsersTable) ID() string { + return "0001_create_users_table" +} + +func (m *Migration0001CreateUsersTable) UpSQL() []string { + switch m.Dialect { + case dbkit.DialectMySQL: + return []string{migration0001CreateUsersTableUpMySQL} + case dbkit.DialectPgx, dbkit.DialectPostgres: + return []string{migration0001CreateUsersTableUpPostgres} + } + return nil +} + +func (m *Migration0001CreateUsersTable) DownSQL() []string { + switch m.Dialect { + case dbkit.DialectMySQL, dbkit.DialectPgx, dbkit.DialectPostgres: + return []string{migration0001CreateUsersTableDown} + } + return nil +} +``` + +**Migration for Creating the Notes Table** + +```go +package main + +import ( + "github.com/acronis/go-dbkit" + "github.com/acronis/go-dbkit/migrate" +) + +const migration0002CreateNotesTableUpMySQL = ` +CREATE TABLE notes ( + id BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, + content TEXT, + user_id BIGINT NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE +); +` + +const migration0002CreateNotesTableUpPostgres = ` +CREATE TABLE notes ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + content TEXT, + user_id BIGINT NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE +); +` + +const migration0002CreateNotesTableDown = ` +DROP TABLE notes; +` + +func NewMigration0002CreateNotesTable(dialect dbkit.Dialect) *migrate.CustomMigration { + var upSQL []string + var downSQL []string + switch dialect { + case dbkit.DialectMySQL: + upSQL = []string{migration0002CreateNotesTableUpMySQL} + downSQL = []string{migration0002CreateNotesTableDown} + case dbkit.DialectPgx, dbkit.DialectPostgres: + upSQL = []string{migration0002CreateNotesTableUpPostgres} + downSQL = []string{migration0002CreateNotesTableDown} + } + return migrate.NewCustomMigration("0002_create_notes_table", upSQL, downSQL, nil, nil) +} +``` + +## License + +Copyright © 2025 Acronis International GmbH. + +Licensed under [MIT License](./../LICENSE). + +[GoDoc]: https://pkg.go.dev/github.com/acronis/go-dbkit/migrate +[GoDoc Widget]: https://godoc.org/github.com/acronis/go-dbkit/migrate?status.svg \ No newline at end of file diff --git a/migrate/examples/embedded-sql-migrations/main.go b/migrate/examples/embedded-sql-migrations/main.go new file mode 100644 index 0000000..f40eff8 --- /dev/null +++ b/migrate/examples/embedded-sql-migrations/main.go @@ -0,0 +1,83 @@ +/* +Copyright © 2025 Acronis International GmbH. + +Released under MIT license. +*/ + +package main + +import ( + "database/sql" + "embed" + "flag" + "fmt" + stdlog "log" + "os" + + "github.com/acronis/go-appkit/log" + _ "github.com/go-sql-driver/mysql" + _ "github.com/jackc/pgx/v5/stdlib" + _ "github.com/lib/pq" + + "github.com/acronis/go-dbkit" + "github.com/acronis/go-dbkit/migrate" +) + +//go:embed mysql/*.sql +//go:embed postgres/*.sql +var migrationFS embed.FS + +func main() { + if err := runMigrations(); err != nil { + stdlog.Fatal(err) + } +} + +func runMigrations() error { + var migrateDown bool + flag.BoolVar(&migrateDown, "down", false, "migrate down") + var driverName string + flag.StringVar(&driverName, "driver", "", "driver name, supported values: mysql, postgres, pgx") + flag.Parse() + + migrationDirection := migrate.MigrationsDirectionUp + if migrateDown { + migrationDirection = migrate.MigrationsDirectionDown + } + + dialect, migrationDirName, err := parseDialectFromDriver(driverName) + if err != nil { + return fmt.Errorf("parse dialect: %w", err) + } + + dbConn, err := sql.Open(driverName, os.Getenv("DB_DSN")) + if err != nil { + return fmt.Errorf("open database: %w", err) + } + + logger, loggerClose := log.NewLogger(&log.Config{Output: log.OutputStderr, Level: log.LevelInfo}) + defer loggerClose() + + migrationManager, err := migrate.NewMigrationsManager(dbConn, dialect, logger) + if err != nil { + return err + } + migrations, err := migrate.LoadAllEmbedFSMigrations(migrationFS, migrationDirName) + if err != nil { + return fmt.Errorf("make embed fs migrations: %w", err) + } + return migrationManager.Run(migrations, migrationDirection) +} + +func parseDialectFromDriver(driverName string) (dialect dbkit.Dialect, migrationDirName string, err error) { + switch driverName { + case "mysql": + return dbkit.DialectMySQL, "mysql", nil + case "postgres": + return dbkit.DialectPostgres, "postgres", nil + case "pgx": + return dbkit.DialectPgx, "postgres", nil + default: + return "", "", fmt.Errorf("unknown driver name: %s", driverName) + } +} diff --git a/migrate/examples/embedded-sql-migrations/mysql/0001_create_users_table.down.sql b/migrate/examples/embedded-sql-migrations/mysql/0001_create_users_table.down.sql new file mode 100644 index 0000000..cc1f647 --- /dev/null +++ b/migrate/examples/embedded-sql-migrations/mysql/0001_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE users; diff --git a/migrate/examples/embedded-sql-migrations/mysql/0001_create_users_table.up.sql b/migrate/examples/embedded-sql-migrations/mysql/0001_create_users_table.up.sql new file mode 100644 index 0000000..892c2df --- /dev/null +++ b/migrate/examples/embedded-sql-migrations/mysql/0001_create_users_table.up.sql @@ -0,0 +1,4 @@ +CREATE TABLE users ( + id BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, + name VARCHAR(255) NOT NULL +); diff --git a/migrate/examples/embedded-sql-migrations/mysql/0002_create_notes_table.down.sql b/migrate/examples/embedded-sql-migrations/mysql/0002_create_notes_table.down.sql new file mode 100644 index 0000000..5a46820 --- /dev/null +++ b/migrate/examples/embedded-sql-migrations/mysql/0002_create_notes_table.down.sql @@ -0,0 +1 @@ +DROP TABLE notes; diff --git a/migrate/examples/embedded-sql-migrations/mysql/0002_create_notes_table.up.sql b/migrate/examples/embedded-sql-migrations/mysql/0002_create_notes_table.up.sql new file mode 100644 index 0000000..98592d5 --- /dev/null +++ b/migrate/examples/embedded-sql-migrations/mysql/0002_create_notes_table.up.sql @@ -0,0 +1,6 @@ +CREATE TABLE notes ( + id BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, + content TEXT, + user_id BIGINT NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE +); diff --git a/migrate/examples/embedded-sql-migrations/postgres/0001_create_users_table.down.sql b/migrate/examples/embedded-sql-migrations/postgres/0001_create_users_table.down.sql new file mode 100644 index 0000000..cc1f647 --- /dev/null +++ b/migrate/examples/embedded-sql-migrations/postgres/0001_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE users; diff --git a/migrate/examples/embedded-sql-migrations/postgres/0001_create_users_table.up.sql b/migrate/examples/embedded-sql-migrations/postgres/0001_create_users_table.up.sql new file mode 100644 index 0000000..4828a24 --- /dev/null +++ b/migrate/examples/embedded-sql-migrations/postgres/0001_create_users_table.up.sql @@ -0,0 +1,4 @@ +CREATE TABLE users ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name VARCHAR(255) NOT NULL +); diff --git a/migrate/examples/embedded-sql-migrations/postgres/0002_create_notes_table.down.sql b/migrate/examples/embedded-sql-migrations/postgres/0002_create_notes_table.down.sql new file mode 100644 index 0000000..5a46820 --- /dev/null +++ b/migrate/examples/embedded-sql-migrations/postgres/0002_create_notes_table.down.sql @@ -0,0 +1 @@ +DROP TABLE notes; diff --git a/migrate/examples/embedded-sql-migrations/postgres/0002_create_notes_table.up.sql b/migrate/examples/embedded-sql-migrations/postgres/0002_create_notes_table.up.sql new file mode 100644 index 0000000..4f75c8d --- /dev/null +++ b/migrate/examples/embedded-sql-migrations/postgres/0002_create_notes_table.up.sql @@ -0,0 +1,6 @@ +CREATE TABLE notes ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + content TEXT, + user_id BIGINT NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE +); diff --git a/migrate/examples/go-sql-migrations/main.go b/migrate/examples/go-sql-migrations/main.go new file mode 100644 index 0000000..ed1228e --- /dev/null +++ b/migrate/examples/go-sql-migrations/main.go @@ -0,0 +1,77 @@ +/* +Copyright © 2025 Acronis International GmbH. + +Released under MIT license. +*/ + +package main + +import ( + "database/sql" + "flag" + "fmt" + stdlog "log" + "os" + + "github.com/acronis/go-appkit/log" + _ "github.com/go-sql-driver/mysql" + _ "github.com/jackc/pgx/v5/stdlib" + _ "github.com/lib/pq" + + "github.com/acronis/go-dbkit" + "github.com/acronis/go-dbkit/migrate" +) + +func main() { + if err := runMigrations(); err != nil { + stdlog.Fatal(err) + } +} + +func runMigrations() error { + var migrateDown bool + flag.BoolVar(&migrateDown, "down", false, "migrate down") + var driverName string + flag.StringVar(&driverName, "driver", "", "driver name, supported values: mysql, postgres, pgx") + flag.Parse() + + migrationDirection := migrate.MigrationsDirectionUp + if migrateDown { + migrationDirection = migrate.MigrationsDirectionDown + } + + dialect, err := parseDialectFromDriver(driverName) + if err != nil { + return fmt.Errorf("parse dialect: %w", err) + } + + dbConn, err := sql.Open(driverName, os.Getenv("DB_DSN")) + if err != nil { + return fmt.Errorf("open database: %w", err) + } + + logger, loggerClose := log.NewLogger(&log.Config{Output: log.OutputStderr, Level: log.LevelInfo}) + defer loggerClose() + + migrationManager, err := migrate.NewMigrationsManager(dbConn, dialect, logger) + if err != nil { + return err + } + return migrationManager.Run([]migrate.Migration{ + NewMigration0001CreateUsersTable(dialect), + NewMigration0002CreateNotesTable(dialect), + }, migrationDirection) +} + +func parseDialectFromDriver(driverName string) (dbkit.Dialect, error) { + switch driverName { + case "mysql": + return dbkit.DialectMySQL, nil + case "postgres": + return dbkit.DialectPostgres, nil + case "pgx": + return dbkit.DialectPgx, nil + default: + return "", fmt.Errorf("unknown driver name: %s", driverName) + } +} diff --git a/migrate/examples/go-sql-migrations/migration_0001_create_users_table.go b/migrate/examples/go-sql-migrations/migration_0001_create_users_table.go new file mode 100644 index 0000000..5d54c9c --- /dev/null +++ b/migrate/examples/go-sql-migrations/migration_0001_create_users_table.go @@ -0,0 +1,60 @@ +/* +Copyright © 2025 Acronis International GmbH. + +Released under MIT license. +*/ + +package main + +import ( + "github.com/acronis/go-dbkit" + "github.com/acronis/go-dbkit/migrate" +) + +const migration0001CreateUsersTableUpMySQL = ` +CREATE TABLE users ( + id BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, + name VARCHAR(255) NOT NULL +); +` + +const migration0001CreateUsersTableUpPostgres = ` +CREATE TABLE users ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name VARCHAR(255) NOT NULL +); +` + +const migration0001CreateUsersTableDown = ` +DROP TABLE users; +` + +type Migration0001CreateUsersTable struct { + *migrate.NullMigration +} + +func NewMigration0001CreateUsersTable(dialect dbkit.Dialect) *Migration0001CreateUsersTable { + return &Migration0001CreateUsersTable{&migrate.NullMigration{Dialect: dialect}} +} + +func (m *Migration0001CreateUsersTable) ID() string { + return "0001_create_users_table" +} + +func (m *Migration0001CreateUsersTable) UpSQL() []string { + switch m.Dialect { + case dbkit.DialectMySQL: + return []string{migration0001CreateUsersTableUpMySQL} + case dbkit.DialectPgx, dbkit.DialectPostgres: + return []string{migration0001CreateUsersTableUpPostgres} + } + return nil +} + +func (m *Migration0001CreateUsersTable) DownSQL() []string { + switch m.Dialect { + case dbkit.DialectMySQL, dbkit.DialectPgx, dbkit.DialectPostgres: + return []string{migration0001CreateUsersTableDown} + } + return nil +} diff --git a/migrate/examples/go-sql-migrations/migration_0002_create_notes_table.go b/migrate/examples/go-sql-migrations/migration_0002_create_notes_table.go new file mode 100644 index 0000000..fdaf0ad --- /dev/null +++ b/migrate/examples/go-sql-migrations/migration_0002_create_notes_table.go @@ -0,0 +1,48 @@ +/* +Copyright © 2025 Acronis International GmbH. + +Released under MIT license. +*/ + +package main + +import ( + "github.com/acronis/go-dbkit" + "github.com/acronis/go-dbkit/migrate" +) + +const migration0002CreateNotesTableUpMySQL = ` +CREATE TABLE notes ( + id BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, + content TEXT, + user_id BIGINT NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE +); +` + +const migration0002CreateNotesTableUpPostgres = ` +CREATE TABLE notes ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + content TEXT, + user_id BIGINT NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE +); +` + +const migration0002CreateNotesTableDown = ` +DROP TABLE notes; +` + +func NewMigration0002CreateNotesTable(dialect dbkit.Dialect) *migrate.CustomMigration { + var upSQL []string + var downSQL []string + switch dialect { + case dbkit.DialectMySQL: + upSQL = []string{migration0002CreateNotesTableUpMySQL} + downSQL = []string{migration0002CreateNotesTableDown} + case dbkit.DialectPgx, dbkit.DialectPostgres: + upSQL = []string{migration0002CreateNotesTableUpPostgres} + downSQL = []string{migration0002CreateNotesTableDown} + } + return migrate.NewCustomMigration("0002_create_notes_table", upSQL, downSQL, nil, nil) +} diff --git a/migrate/migrations.go b/migrate/migrations.go index 8466951..99afcca 100644 --- a/migrate/migrations.go +++ b/migrate/migrations.go @@ -9,7 +9,11 @@ package migrate import ( "database/sql" + "embed" "fmt" + "path/filepath" + "sort" + "strings" "time" "github.com/acronis/go-appkit/log" @@ -18,7 +22,7 @@ import ( "github.com/acronis/go-dbkit" ) -// MigrationsTableName contains name of table in a database that stores applied migrations. +// MigrationsTableName contains the name of table in a database that stores applied migrations. const MigrationsTableName = "migrations" // MigrationsDirection defines possible values for direction of database migrations. @@ -44,7 +48,7 @@ type Migration interface { DownFn() func(tx *sql.Tx) error // Not supported yet. } -// RawMigrator is an interface which allows overwrite default generate mechanism for full control on migrations. +// RawMigrator is an interface that allows to overwrite default generate mechanism for full control on migrations. // Uses sql-migrate migration structure. type RawMigrator interface { RawMigration(m Migration) (*migrate.Migration, error) @@ -173,8 +177,8 @@ func (mm *MigrationsManager) Run(migrations []Migration, direction MigrationsDir } // convertMigration converts migration to internal sql-migrate format. -// If migration implements RawMigrator interface then RawMigration function is used. -// If migration implements TxDisabler interface then it may be not in transaction. +// If migration implements RawMigrator interface, then RawMigration function is used. +// If migration implements TxDisabler interface, then it may be not in transaction. func convertMigration(m Migration) (*migrate.Migration, error) { if migrator, ok := m.(RawMigrator); ok { raw, err := migrator.RawMigration(m) @@ -282,3 +286,83 @@ func (ms *MigrationStatus) LastAppliedMigration() (appliedMig AppliedMigration, } return ms.AppliedMigrations[len(ms.AppliedMigrations)-1], true } + +// LoadAllEmbedFSMigrations loads all migrations from the embed.FS directory. +func LoadAllEmbedFSMigrations(fs embed.FS, dirName string) ([]Migration, error) { + files, err := fs.ReadDir(dirName) + if err != nil { + return nil, fmt.Errorf("read migrations directory %s: %w", dirName, err) + } + + migrationsMap := make(map[string][2]string) + for _, file := range files { + if file.IsDir() { + continue + } + var migrationID string + var nameIdx int + switch { + case strings.HasSuffix(file.Name(), ".up.sql"): + migrationID = strings.TrimSuffix(file.Name(), ".up.sql") + nameIdx = 0 + case strings.HasSuffix(file.Name(), ".down.sql"): + migrationID = strings.TrimSuffix(file.Name(), ".down.sql") + nameIdx = 1 + default: + return nil, fmt.Errorf("migration file should have .up.sql or .down.sql suffix, got %s", file.Name()) + } + names := migrationsMap[migrationID] + names[nameIdx] = file.Name() + migrationsMap[migrationID] = names + } + + migrations := make([]Migration, 0, len(migrationsMap)) + for migrationID, names := range migrationsMap { + if names[0] == "" { + return nil, fmt.Errorf("%s migration up file is missing", migrationID) + } + if names[1] == "" { + return nil, fmt.Errorf("%s migration down file is missing", migrationID) + } + var upSQL []byte + if upSQL, err = fs.ReadFile(filepath.Join(dirName, names[0])); err != nil { + return nil, err + } + var downSQL []byte + if downSQL, err = fs.ReadFile(filepath.Join(dirName, names[1])); err != nil { + return nil, err + } + migrations = append(migrations, &CustomMigration{ + id: migrationID, + upSQL: []string{string(upSQL)}, + downSQL: []string{string(downSQL)}, + }) + } + + sort.Slice(migrations, func(i, j int) bool { + return migrations[i].ID() < migrations[j].ID() + }) + + return migrations, nil +} + +// LoadEmbedFSMigrations loads migrations with specified IDs from the embed.FS directory. +func LoadEmbedFSMigrations(fs embed.FS, dirName string, migrationIDs []string) ([]Migration, error) { + migrations := make([]Migration, 0, len(migrationIDs)) + for _, migrationID := range migrationIDs { + upSQL, err := fs.ReadFile(filepath.Join(dirName, fmt.Sprintf("%s.up.sql", migrationID))) + if err != nil { + return nil, err + } + downSQL, err := fs.ReadFile(filepath.Join(dirName, fmt.Sprintf("%s.down.sql", migrationID))) + if err != nil { + return nil, err + } + migrations = append(migrations, &CustomMigration{ + id: migrationID, + upSQL: []string{string(upSQL)}, + downSQL: []string{string(downSQL)}, + }) + } + return migrations, nil +} diff --git a/migrate/migrations_test.go b/migrate/migrations_test.go index f805674..216d7fd 100644 --- a/migrate/migrations_test.go +++ b/migrate/migrations_test.go @@ -9,17 +9,18 @@ package migrate import ( "bytes" "database/sql" + "embed" "fmt" "io" "testing" "time" "github.com/acronis/go-appkit/log/logtest" + _ "github.com/mattn/go-sqlite3" migrate "github.com/rubenv/sql-migrate" "github.com/stretchr/testify/require" "github.com/acronis/go-dbkit" - _ "github.com/acronis/go-dbkit/sqlite" ) type testMigration00001CreateTables struct { @@ -234,20 +235,15 @@ func TestCreationMigrationManagerWithOpts(t *testing.T) { require.NoError(t, err) defer requireNoErrOnClose(t, dbConn) - migMngr, err := NewMigrationsManagerWithOpts( - dbConn, - dbkit.DialectSQLite, - logtest.NewLogger(), - MigrationsManagerOpts{TableName: tableName}, - ) + migMngr, err := NewMigrationsManagerWithOpts(dbConn, dbkit.DialectSQLite, logtest.NewLogger(), + MigrationsManagerOpts{TableName: tableName}) require.NoError(t, err) - require.Equal(t, tableName, migMngr.migSet.TableName) migrations := []Migration{newTestMigration00001CreateTables(), newTestMigration00002SeedTabled()} var rowsNum int - // Table doesn't exist before migrations. + // The table doesn't exist before migrations. require.Error(t, dbConn.QueryRow("select count(*) from custom_migrations").Scan(&rowsNum)) // Run migrations. @@ -316,3 +312,166 @@ func TestMigrationsManager_supportRawMigration(t *testing.T) { require.NoError(t, migMngr.RunLimit(migrations, MigrationsDirectionDown, 1)) requireMigrationsApplied(t, dbConn, true, 0, 0) } + +//go:embed testdata/sqlite/*.sql +//go:embed testdata/missing-down-file/*.sql +//go:embed testdata/missing-up-file/*.sql +//go:embed testdata/invalid-suffix/*.sql +var testFS embed.FS + +func TestAllLoadEmbedFSMigrations(t *testing.T) { + tests := []struct { + name string + fs embed.FS + dirName string + wantErrMsg string + expectedIDs []string + }{ + { + name: "valid migrations", + fs: testFS, + dirName: "testdata/sqlite", + expectedIDs: []string{"0001_create_users_table", "0002_create_notes_table", "0003_seed_tables"}, + }, + { + name: "missing up file", + fs: testFS, + dirName: "testdata/missing-up-file", + wantErrMsg: "0001_create_users_table migration up file is missing", + }, + { + name: "missing down file", + fs: testFS, + dirName: "testdata/missing-down-file", + wantErrMsg: "0001_create_users_table migration down file is missing", + }, + { + name: "invalid suffix", + fs: testFS, + dirName: "testdata/invalid-suffix", + wantErrMsg: "migration file should have .up.sql or .down.sql suffix, got 0001_create_users_table.sql", + }, + { + name: "non-existent directory", + fs: testFS, + dirName: "testdata/non-existent", + wantErrMsg: "read migrations directory testdata/non-existent: open testdata/non-existent: file does not exist", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + migrations, err := LoadAllEmbedFSMigrations(tt.fs, tt.dirName) + if tt.wantErrMsg != "" { + require.EqualError(t, err, tt.wantErrMsg) + return + } + require.NoError(t, err) + require.Len(t, migrations, len(tt.expectedIDs)) + for i, migration := range migrations { + require.Equal(t, tt.expectedIDs[i], migration.ID()) + } + + dbConn, err := sql.Open("sqlite3", "file::memory:?cache=shared") + require.NoError(t, err) + defer requireNoErrOnClose(t, dbConn) + + migManager, err := NewMigrationsManager(dbConn, dbkit.DialectSQLite, logtest.NewLogger()) + require.NoError(t, err) + require.NoError(t, migManager.Run(migrations, MigrationsDirectionUp)) + + var usersCount int + require.NoError(t, dbConn.QueryRow("select count(*) from users").Scan(&usersCount)) + require.Equal(t, 3, usersCount) + var notesCount int + require.NoError(t, dbConn.QueryRow("select count(*) from notes").Scan(¬esCount)) + require.Equal(t, 2, notesCount) + + migStatus, err := migManager.Status() + require.NoError(t, err) + appliedIDs := make([]string, 0, len(migStatus.AppliedMigrations)) + for _, mig := range migStatus.AppliedMigrations { + appliedIDs = append(appliedIDs, mig.ID) + } + require.Equal(t, tt.expectedIDs, appliedIDs) + }) + } +} + +func TestLoadEmbedFSMigrations(t *testing.T) { + tests := []struct { + name string + fs embed.FS + dirName string + migrationIDs []string + wantErrMsg string + expectedIDs []string + }{ + { + name: "valid migrations", + fs: testFS, + dirName: "testdata/sqlite", + migrationIDs: []string{"0001_create_users_table", "0002_create_notes_table"}, + expectedIDs: []string{"0001_create_users_table", "0002_create_notes_table"}, + }, + { + name: "missing up file", + fs: testFS, + dirName: "testdata/missing-up-file", + migrationIDs: []string{"0001_create_users_table"}, + wantErrMsg: "open testdata/missing-up-file/0001_create_users_table.up.sql: file does not exist", + }, + { + name: "missing down file", + fs: testFS, + dirName: "testdata/missing-down-file", + migrationIDs: []string{"0001_create_users_table"}, + wantErrMsg: "open testdata/missing-down-file/0001_create_users_table.down.sql: file does not exist", + }, + { + name: "invalid migration ID", + fs: testFS, + dirName: "testdata/sqlite", + migrationIDs: []string{"invalid_migration_id"}, + wantErrMsg: "open testdata/sqlite/invalid_migration_id.up.sql: file does not exist", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + migrations, err := LoadEmbedFSMigrations(tt.fs, tt.dirName, tt.migrationIDs) + if tt.wantErrMsg != "" { + require.EqualError(t, err, tt.wantErrMsg) + return + } + require.NoError(t, err) + require.Len(t, migrations, len(tt.expectedIDs)) + for i, migration := range migrations { + require.Equal(t, tt.expectedIDs[i], migration.ID()) + } + + dbConn, err := sql.Open("sqlite3", "file::memory:?cache=shared") + require.NoError(t, err) + defer requireNoErrOnClose(t, dbConn) + + migManager, err := NewMigrationsManager(dbConn, dbkit.DialectSQLite, logtest.NewLogger()) + require.NoError(t, err) + require.NoError(t, migManager.Run(migrations, MigrationsDirectionUp)) + + var usersCount int + require.NoError(t, dbConn.QueryRow("select count(*) from users").Scan(&usersCount)) + require.Equal(t, 0, usersCount) + var notesCount int + require.NoError(t, dbConn.QueryRow("select count(*) from notes").Scan(¬esCount)) + require.Equal(t, 0, notesCount) + + migStatus, err := migManager.Status() + require.NoError(t, err) + appliedIDs := make([]string, 0, len(migStatus.AppliedMigrations)) + for _, mig := range migStatus.AppliedMigrations { + appliedIDs = append(appliedIDs, mig.ID) + } + require.Equal(t, tt.expectedIDs, appliedIDs) + }) + } +} diff --git a/migrate/testdata/invalid-suffix/0001_create_users_table.sql b/migrate/testdata/invalid-suffix/0001_create_users_table.sql new file mode 100644 index 0000000..a7601bb --- /dev/null +++ b/migrate/testdata/invalid-suffix/0001_create_users_table.sql @@ -0,0 +1,4 @@ +CREATE TABLE users ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL +); diff --git a/migrate/testdata/missing-down-file/0001_create_users_table.up.sql b/migrate/testdata/missing-down-file/0001_create_users_table.up.sql new file mode 100644 index 0000000..a7601bb --- /dev/null +++ b/migrate/testdata/missing-down-file/0001_create_users_table.up.sql @@ -0,0 +1,4 @@ +CREATE TABLE users ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL +); diff --git a/migrate/testdata/missing-up-file/0001_create_users_table.down.sql b/migrate/testdata/missing-up-file/0001_create_users_table.down.sql new file mode 100644 index 0000000..cc1f647 --- /dev/null +++ b/migrate/testdata/missing-up-file/0001_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE users; diff --git a/migrate/testdata/sqlite/0001_create_users_table.down.sql b/migrate/testdata/sqlite/0001_create_users_table.down.sql new file mode 100644 index 0000000..cc1f647 --- /dev/null +++ b/migrate/testdata/sqlite/0001_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE users; diff --git a/migrate/testdata/sqlite/0001_create_users_table.up.sql b/migrate/testdata/sqlite/0001_create_users_table.up.sql new file mode 100644 index 0000000..a7601bb --- /dev/null +++ b/migrate/testdata/sqlite/0001_create_users_table.up.sql @@ -0,0 +1,4 @@ +CREATE TABLE users ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL +); diff --git a/migrate/testdata/sqlite/0002_create_notes_table.down.sql b/migrate/testdata/sqlite/0002_create_notes_table.down.sql new file mode 100644 index 0000000..5a46820 --- /dev/null +++ b/migrate/testdata/sqlite/0002_create_notes_table.down.sql @@ -0,0 +1 @@ +DROP TABLE notes; diff --git a/migrate/testdata/sqlite/0002_create_notes_table.up.sql b/migrate/testdata/sqlite/0002_create_notes_table.up.sql new file mode 100644 index 0000000..649cc51 --- /dev/null +++ b/migrate/testdata/sqlite/0002_create_notes_table.up.sql @@ -0,0 +1,6 @@ +CREATE TABLE notes ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + content TEXT, + user_id INTEGER NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE +); diff --git a/migrate/testdata/sqlite/0003_seed_tables.down.sql b/migrate/testdata/sqlite/0003_seed_tables.down.sql new file mode 100644 index 0000000..0a18b74 --- /dev/null +++ b/migrate/testdata/sqlite/0003_seed_tables.down.sql @@ -0,0 +1,2 @@ +DELETE FROM notes; +DELETE FROM users; diff --git a/migrate/testdata/sqlite/0003_seed_tables.up.sql b/migrate/testdata/sqlite/0003_seed_tables.up.sql new file mode 100644 index 0000000..6e79854 --- /dev/null +++ b/migrate/testdata/sqlite/0003_seed_tables.up.sql @@ -0,0 +1,2 @@ +INSERT INTO users(name) VALUES("Alice"), ("Bob"), ("Charlie"); +INSERT INTO notes(content, user_id) VALUES("Note 1", 1), ("Note 2", 2);