Documentation.

Author: AlphaOne1

README.md

@@ -65,15 +65,161 @@
 DMorph
 ======
 
-*DMorph* is a database migration library.
+*DMorph* is a database migration library. Programs that use a database and have to preserve the data
+between versions can utilize *DMorph* to apply the necessary migration steps. If a program can
+afford to lose all data between version upgrades, this library is not necessary.
 
 Includes direct support for the following relational database management systems:
 
 * [IBM Db2](https://www.ibm.com/db2/)
-* [Oracle Database](https://www.oracle.com/database/)
 * [Microsoft SQL Server](https://www.microsoft.com/sql-server)
 * [MySQL](https://www.mysql.com/) & [MariaDB](https://mariadb.org/)
+* [Oracle Database](https://www.oracle.com/database/)
 * [PostgreSQL](https://www.postgresql.org)
 * [SQLite](https://www.sqlite.org)
 
 Additional database management systems can be included providing the necessary queries.
+
+
+Installation
+------------
+
+To install *DMorph*, you can use the following command:
+
+```bash
+$ go get github.com/AlphaOne1/dmorph
+```
+
+
+Getting Started
+---------------
+
+*DMorph* applies migrations to a database. A migration is a series of steps, defined either in an
+SQL file or programmatically.
+
+
+### Migration from File
+
+A typical migration file consists of a sequence of SQL statements. Each statement needs to be
+finalized with a semicolon `;`. If a semicolon is found alone at the beginning of a line, all
+previous statements, that were not yet executed, are executed in one call to Exec in a
+transaction. A migration is executed completely inside of a transaction. If any of the steps of
+a migration fails, a rollback is issued and the process stops. Take care, that not all database
+management systems offer a rollback of DDL (CREATE, DROP, ...) statements.
+
+An example for a migration inside a file `01_base_tables` is as follows:
+
+```sqlite
+CREATE TABLE tab0 (
+    id string PRIMARY KEY
+)
+;
+
+CREATE TABLE tab1 (
+    id string PRIMARY KEY
+)
+;
+```
+
+It can be applied to an already open database with the following snipped:
+
+```go
+package testprog
+
+import (
+    "database/sql"
+    "github.com/AlphaOne1/dmorph"
+)
+
+func migrate(db *sql.DB) error {
+    return dmorph.Run(db,
+        dmorph.WithDialect(dmorph.DialectSQLite()),
+        dmorph.WithMigrationFromFile("01_base_tables.sql"))
+}
+
+...
+```
+
+In this example just one file is used, the `WithMigrationFromFile` can be given multiple times.
+Migrations are executed in alphabetical order of their key. For files the key is the file's name.
+
+
+### Migrations from Folder
+
+As normally multiple migrations are to be executed, they can be assembled in a folder and then
+executed together. As stated before, the order of multiple files is determined by their
+alphabetically ordered name.
+
+Taken the example from [above](#migration-from-file), split into two files, like prepared in
+[testData/](testData/).
+
+```go
+package testprog
+
+import (
+    "database/sql"
+    _ "embed"
+    "io/fs"
+    "github.com/AlphaOne1/dmorph"
+)
+
+//go:embed testData
+var migrationFS embed.FS
+
+func migrate(db *sql.DB) error {
+    sub, subErr := fs.Sub(migrationFS, "testData") 
+    
+    if subErr != nil {
+        return subErr
+	}
+    
+    return dmorph.Run(db,
+        dmorph.WithDialect(dmorph.DialectSQLite()),
+        dmorph.WithMigrationsFromFS(sub.(fs.ReadDirFS)))
+}
+
+...
+```
+
+### Programmatic Migration
+
+Sometimes SQL alone is not sufficient to achieve the migration desired. Maybe the data needs to be
+programmatically changed, checked or otherwise processed. For *DMorph* a migration is presented as
+an interface:
+
+```go
+type Migration interface {
+    Key() string              // identifier, used for ordering
+    Migrate(tx *sql.Tx) error // migration functionality
+}
+```
+
+The `WithMigrationFromF...` family of options constructs these migrations for convenience. An example
+migration fulfilling this interface could look like this:
+
+```go
+type CustomMigration struct {}
+
+func (m *CustomMigration) Key() string {
+    return "0001_custom"
+}
+
+func (m *CustomMigration) Migrate(tx *sql.Tx) error {
+    _, err := tx.Exec(`CREATE TABLE tab0(id INTEGER PRIMARY KEY)`)
+    return err
+}
+```
+
+Inside of the `Migrate` function the transaction state should not be modified.
+`Commit` and `Rollback` are handled by *DMorph* as needed. As seen in the example, a potentiel error
+is returned plain to the caller.
+
+This newly created migration can then be passed to *DMorph* as follows:
+
+```go
+func migrate(db *sql.DB) error {
+    return dmorph.Run(db,
+        dmorph.WithDialect(dmorph.DialectSQLite()),
+        dmorph.WithMigration(CustomMigration{}))
+}
+```

dialect_db2.go

@@ -3,6 +3,7 @@
 
 package dmorph
 
+// DialectDB2 returns a Dialect configured for DB2 databases.
 func DialectDB2() BaseDialect {
 	return BaseDialect{
 		CreateTemplate: `

dialect_mssql.go

@@ -3,6 +3,7 @@
 
 package dmorph
 
+// DialectMSSQL returns a Dialect configured for Microsoft SQL Server databases.
 func DialectMSSQL() BaseDialect {
 	return BaseDialect{
 		CreateTemplate: `

dialect_mysql.go

@@ -3,6 +3,7 @@
 
 package dmorph
 
+// DialectMySQL returns a Dialect configured for MySQL databases.
 func DialectMySQL() BaseDialect {
 	return BaseDialect{
 		CreateTemplate: "CREATE TABLE IF NOT EXISTS `%s`" + ` (

dialect_oracle.go

@@ -3,6 +3,7 @@
 
 package dmorph
 
+// DialectOracle returns a Dialect configured for Oracle Database.
 func DialectOracle() BaseDialect {
 	return BaseDialect{
 		CreateTemplate: `

dialect_postgres.go

@@ -3,6 +3,7 @@
 
 package dmorph
 
+// DialectPostgres returns a Dialect configured for Postgres databases.
 func DialectPostgres() BaseDialect {
 	return BaseDialect{
 		CreateTemplate: `

dialect_sqlite.go

@@ -3,6 +3,7 @@
 
 package dmorph
 
+// DialectSQLite returns a Dialect configured for SQLite databases.
 func DialectSQLite() BaseDialect {
 	return BaseDialect{
 		CreateTemplate: `

dialects.go

@@ -9,12 +9,16 @@ import (
 	"fmt"
 )
 
+// BaseDialect is a convenience type for databases that manage the necessary operations solely using
+// queries. Defining the CreateTemplate, AppliedTemplate and RegisterTemplate enables the BaseDialect to
+// perform all the necessary operation to fulfill the Dialect interface.
 type BaseDialect struct {
 	CreateTemplate   string
 	AppliedTemplate  string
 	RegisterTemplate string
 }
 
+// EnsureMigrationTableExists ensures that the migration table, saving the applied migrations ids, exists.
 func (b BaseDialect) EnsureMigrationTableExists(db *sql.DB, tableName string) error {
 	tx, err := db.Begin()
 
@@ -39,6 +43,7 @@ func (b BaseDialect) EnsureMigrationTableExists(db *sql.DB, tableName string) er
 	return nil
 }
 
+// AppliedMigrations gets the already applied migrations from the database, ordered by application date.
 func (b BaseDialect) AppliedMigrations(db *sql.DB, tableName string) ([]string, error) {
 	rows, rowsErr := db.Query(fmt.Sprintf(b.AppliedTemplate, tableName))
 
@@ -61,6 +66,7 @@ func (b BaseDialect) AppliedMigrations(db *sql.DB, tableName string) ([]string,
 	return result, errors.Join(rows.Err(), scanErr)
 }
 
+// RegisterMigration registers a migration in the migration table.
 func (b BaseDialect) RegisterMigration(tx *sql.Tx, id string, tableName string) error {
 	_, err := tx.Exec(fmt.Sprintf(b.RegisterTemplate, tableName),
 		sql.Named("id", id))

file_migration.go

@@ -13,20 +13,24 @@ import (
 	"os"
 )
 
+// FileMigration implements the Migration interface. It helps to apply migrations from a file or fs.FS.
 type FileMigration struct {
 	Name          string
 	FS            fs.FS
 	migrationFunc func(tx *sql.Tx, migration string) error
 }
 
+// Key returns the key of the migration to register in the migration table.
 func (f FileMigration) Key() string {
 	return f.Name
 }
 
+// Migrate executes the migration on the given transaction.
 func (f FileMigration) Migrate(tx *sql.Tx) error {
 	return f.migrationFunc(tx, f.Name)
 }
 
+// WithMigrationFromFile generates a FileMigration that will run the content of the given file.
 func WithMigrationFromFile(name string) MorphOption {
 	return func(morpher *Morpher) error {
 		morpher.Migrations = append(morpher.Migrations, FileMigration{
@@ -40,14 +44,16 @@ func WithMigrationFromFile(name string) MorphOption {
 
 				defer func() { _ = m.Close() }()
 
-				return applyFileSteps(tx, m, migration, morpher.Log)
+				return applyStepsStream(tx, m, migration, morpher.Log)
 			},
 		})
 
 		return nil
 	}
 }
 
+// WithMigrationFromFileFS generates a FileMigration that will run the content of the given file from the
+// given filesystem.
 func WithMigrationFromFileFS(name string, dir fs.FS) MorphOption {
 	return func(morpher *Morpher) error {
 		morpher.Migrations = append(morpher.Migrations, migrationFromFileFS(name, dir, morpher.Log))
@@ -56,26 +62,27 @@ func WithMigrationFromFileFS(name string, dir fs.FS) MorphOption {
 	}
 }
 
+// WithMigrationsFromFS generates a FileMigration that will run all migration scripts of the files in the given
+// filesystem.
 func WithMigrationsFromFS(d fs.ReadDirFS) MorphOption {
 	return func(morpher *Morpher) error {
 		dirEntry, err := d.ReadDir(".")
 
-		if err != nil {
-			return err
-		}
-
-		for _, entry := range dirEntry {
-			morpher.Log.Info("entry", slog.String("name", entry.Name()))
-			if entry.Type().IsRegular() {
-				morpher.Migrations = append(morpher.Migrations,
-					migrationFromFileFS(entry.Name(), d, morpher.Log))
+		if err == nil {
+			for _, entry := range dirEntry {
+				morpher.Log.Info("entry", slog.String("name", entry.Name()))
+				if entry.Type().IsRegular() {
+					morpher.Migrations = append(morpher.Migrations,
+						migrationFromFileFS(entry.Name(), d, morpher.Log))
+				}
 			}
 		}
 
-		return nil
+		return err
 	}
 }
 
+// migrationFromFileFS creates a FileMigration instance for a specific migration file from an fs.FS directory.
 func migrationFromFileFS(name string, dir fs.FS, log *slog.Logger) FileMigration {
 	return FileMigration{
 		Name: name,
@@ -89,17 +96,20 @@ func migrationFromFileFS(name string, dir fs.FS, log *slog.Logger) FileMigration
 
 			defer func() { _ = m.Close() }()
 
-			return applyFileSteps(tx, m, migration, log)
+			return applyStepsStream(tx, m, migration, log)
 		},
 	}
 }
 
-func applyFileSteps(tx *sql.Tx, r io.Reader, id string, log *slog.Logger) error {
+// applyStepsStream executes database migration steps read from an io.Reader, separated by semicolons, in a transaction.
+// Returns the corresponding error if any step execution fails.
+func applyStepsStream(tx *sql.Tx, r io.Reader, id string, log *slog.Logger) error {
 	buf := bytes.Buffer{}
 
 	scanner := bufio.NewScanner(r)
+	var i int
 
-	for i := 0; scanner.Scan(); {
+	for i = 0; scanner.Scan(); {
 		buf.Write(scanner.Bytes())
 
 		if scanner.Text() == ";" {
@@ -116,5 +126,17 @@ func applyFileSteps(tx *sql.Tx, r io.Reader, id string, log *slog.Logger) error
 		}
 	}
 
+	// cleanup after, for final statement without the closing ; on a new line
+	if buf.Len() > 0 {
+		log.Info("migration step",
+			slog.String("id", id),
+			slog.Int("step", i),
+		)
+
+		if _, err := tx.Exec(buf.String()); err != nil {
+			return err
+		}
+	}
+
 	return scanner.Err()
 }