Merge branch 'master' into add-galera-support

Author: Gonlo2

.github/workflows/ci.yml

@@ -0,0 +1,20 @@
+name: CI
+
+on: [pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Set up Go 1.12
+      uses: actions/setup-go@v1
+      with:
+        version: 1.12
+      id: go
+
+    - name: Build
+      run: script/cibuild

.github/workflows/replica-tests.yml

@@ -0,0 +1,20 @@
+name: migration tests
+
+on: [pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Set up Go 1.12
+      uses: actions/setup-go@v1
+      with:
+        version: 1.12
+      id: go
+
+    - name: migration tests
+      run:  script/cibuild-gh-ost-replica-tests

.travis.yml

@@ -2,12 +2,14 @@
 language: go
 
 go:
-  - "1.9"
-  - "1.10"
+  - "1.12.x"
 
 os:
   - linux
 
+services:
+  - mysql
+
 env:
 - MYSQL_USER=root
 - CURRENT_CI_ENV=travis

Dockerfile.test

@@ -0,0 +1,11 @@
+FROM golang:1.12.1
+LABEL maintainer="github@github.com"
+
+RUN apt-get update
+RUN apt-get install -y lsb-release
+RUN rm -rf /var/lib/apt/lists/*
+
+COPY . /go/src/github.com/github/gh-ost
+WORKDIR /go/src/github.com/github/gh-ost
+
+CMD ["script/test"]

README.md

@@ -94,7 +94,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d
 
 [Download latest release here](https://github.com/github/gh-ost/releases/latest)
 
-`gh-ost` is a Go project; it is built with Go `1.9` and above. To build on your own, use either:
+`gh-ost` is a Go project; it is built with Go `1.12` and above. To build on your own, use either:
 - [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary.
 - [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost`
 

build.sh

@@ -18,10 +18,8 @@ function build {
   GOOS=$3
   GOARCH=$4
 
-
-
-  if ! go version | egrep -q 'go(1[.]9|1[.]1[0-9])' ; then
-    echo "go version is too low. Must use 1.9 or above"
+  if ! go version | egrep -q 'go(1\.1[234])' ; then
+    echo "go version must be 1.12 or above"
     exit 1
   fi
 

docker-compose.yml

@@ -0,0 +1,7 @@
+version: "3.5"
+services:
+  app:
+    image: app
+    build:
+      context: .
+      dockerfile: Dockerfile.test

go/logic/applier.go

@@ -485,9 +485,12 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected
 		}
 		defer tx.Rollback()
 		sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s'`, this.migrationContext.ApplierTimeZone)
+		sqlModeAddendum := `,NO_AUTO_VALUE_ON_ZERO`
 		if !this.migrationContext.SkipStrictMode {
-			sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')"
+			sqlModeAddendum = fmt.Sprintf("%s,STRICT_ALL_TABLES", sqlModeAddendum)
 		}
+		sessionQuery = fmt.Sprintf("%s, sql_mode = CONCAT(@@session.sql_mode, ',%s')", sessionQuery, sqlModeAddendum)
+
 		if _, err := tx.Exec(sessionQuery); err != nil {
 			return nil, err
 		}
@@ -1270,63 +1273,6 @@ func (this *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) (result
 	return append(results, newDmlBuildResultError(fmt.Errorf("Unknown dml event type: %+v", dmlEvent.DML)))
 }
 
-// ApplyDMLEventQuery writes an entry to the ghost table, in response to an intercepted
-// original-table binlog event
-func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error {
-	for _, buildResult := range this.buildDMLEventQuery(dmlEvent) {
-		if buildResult.err != nil {
-			return buildResult.err
-		}
-		// TODO The below is in preparation for transactional writes on the ghost tables.
-		// Such writes would be, for example:
-		// - prepended with sql_mode setup
-		// - prepended with time zone setup
-		// - prepended with SET SQL_LOG_BIN=0
-		// - prepended with SET FK_CHECKS=0
-		// etc.
-		//
-		// a known problem: https://github.com/golang/go/issues/9373 -- bitint unsigned values, not supported in database/sql
-		// is solved by silently converting unsigned bigints to string values.
-		//
-
-		err := func() error {
-			tx, err := this.db.Begin()
-			if err != nil {
-				return err
-			}
-			rollback := func(err error) error {
-				tx.Rollback()
-				return err
-			}
-			sessionQuery := fmt.Sprintf("SET SESSION time_zone = '+00:00'")
-			if !this.migrationContext.SkipStrictMode {
-				sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')"
-			}
-			if _, err := tx.Exec(sessionQuery); err != nil {
-				return rollback(err)
-			}
-			if _, err := tx.Exec(buildResult.query, buildResult.args...); err != nil {
-				return rollback(err)
-			}
-			if err := tx.Commit(); err != nil {
-				return err
-			}
-			return nil
-		}()
-
-		if err != nil {
-			err = fmt.Errorf("%s; query=%s; args=%+v", err.Error(), buildResult.query, buildResult.args)
-			return log.Errore(err)
-		}
-		// no error
-		atomic.AddInt64(&this.migrationContext.TotalDMLEventsApplied, 1)
-		if this.migrationContext.CountTableRows {
-			atomic.AddInt64(&this.migrationContext.RowsDeltaEstimate, buildResult.rowsDelta)
-		}
-	}
-	return nil
-}
-
 // ApplyDMLEventQueries applies multiple DML queries onto the _ghost_ table
 func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) error {
 
@@ -1344,9 +1290,13 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent))
 		}
 
 		sessionQuery := "SET SESSION time_zone = '+00:00'"
+
+		sqlModeAddendum := `,NO_AUTO_VALUE_ON_ZERO`
 		if !this.migrationContext.SkipStrictMode {
-			sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')"
+			sqlModeAddendum = fmt.Sprintf("%s,STRICT_ALL_TABLES", sqlModeAddendum)
 		}
+		sessionQuery = fmt.Sprintf("%s, sql_mode = CONCAT(@@session.sql_mode, ',%s')", sessionQuery, sqlModeAddendum)
+
 		if _, err := tx.Exec(sessionQuery); err != nil {
 			return rollback(err)
 		}

go/logic/server.go

@@ -144,7 +144,7 @@ func (this *Server) applyServerCommand(command string, writer *bufio.Writer) (pr
 	switch command {
 	case "help":
 		{
-			fmt.Fprintln(writer, `available commands:
+			fmt.Fprint(writer, `available commands:
 status                               # Print a detailed status message
 sup                                  # Print a short status message
 coordinates													 # Print the currently inspected coordinates
@@ -341,7 +341,7 @@ help                                 # This message
 				err := fmt.Errorf("User commanded 'panic' on %s, but migrated table is %s; ignoring request.", arg, this.migrationContext.OriginalTableName)
 				return NoPrintStatusRule, err
 			}
-			err := fmt.Errorf("User commanded 'panic'. I will now panic, without cleanup. PANIC!")
+			err := fmt.Errorf("User commanded 'panic'. The migration will be aborted without cleanup. Please drop the gh-ost tables before trying again.")
 			this.migrationContext.PanicAbort <- err
 			return NoPrintStatusRule, err
 		}

localtests/autoinc-zero-value/create.sql

@@ -0,0 +1,9 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  primary key(id)
+) auto_increment=1;
+
+set session sql_mode='NO_AUTO_VALUE_ON_ZERO';
+insert into gh_ost_test values (0, 23);