From 7787aa91dc3e6dd3f9c30948f65e6bba4926195b Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Tue, 18 Feb 2025 11:07:41 +0100 Subject: [PATCH 01/11] Execute hook on every batch insert retry Co-authored-by: Bastian Bartmann --- go/logic/hooks.go | 6 +++++ go/logic/migrator.go | 54 +++++++++++++++++++++++++++----------------- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 2543f8e9a..68626c9f3 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -28,6 +28,7 @@ const ( onInteractiveCommand = "gh-ost-on-interactive-command" onSuccess = "gh-ost-on-success" onFailure = "gh-ost-on-failure" + onBatchCopyRetry = "gh-ost-on-batch-copy-retry" onStatus = "gh-ost-on-status" onStopReplication = "gh-ost-on-stop-replication" onStartReplication = "gh-ost-on-start-replication" @@ -77,6 +78,7 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [ // executeHook executes a command, and sets relevant environment variables // combined output & error are printed to the configured writer. func (this *HooksExecutor) executeHook(hook string, extraVariables ...string) error { + this.migrationContext.Log.Infof("executing hook: %+v", hook) cmd := exec.Command(hook) cmd.Env = this.applyEnvironmentVariables(extraVariables...) @@ -123,6 +125,10 @@ func (this *HooksExecutor) onBeforeRowCopy() error { return this.executeHooks(onBeforeRowCopy) } +func (this *HooksExecutor) onBatchCopyRetry() error { + return this.executeHooks(onBatchCopyRetry) +} + func (this *HooksExecutor) onRowCopyComplete() error { return this.executeHooks(onRowCopyComplete) } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 09de0977b..5b9c4dd24 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -130,6 +130,18 @@ func (this *Migrator) sleepWhileTrue(operation func() (bool, error)) error { } } +func (this *Migrator) retryBatchCopyWithHooks(operation func() error, notFatalHint ...bool) (err error) { + wrappedOperation := func() error { + if err := operation(); err != nil { + this.hooksExecutor.onBatchCopyRetry() + return err + } + return nil + } + + return this.retryOperation(wrappedOperation, notFatalHint...) +} + // retryOperation attempts up to `count` attempts at running given function, // exiting as soon as it returns with non-error. func (this *Migrator) retryOperation(operation func() error, notFatalHint ...bool) (err error) { @@ -1232,28 +1244,28 @@ func (this *Migrator) iterateChunks() error { return nil } copyRowsFunc := func() error { - if atomic.LoadInt64(&this.rowCopyCompleteFlag) == 1 || atomic.LoadInt64(&hasNoFurtherRangeFlag) == 1 { - // Done. - // There's another such check down the line - return nil - } - - // When hasFurtherRange is false, original table might be write locked and CalculateNextIterationRangeEndValues would hangs forever - - hasFurtherRange := false - expectedRangeSize := int64(0) - if err := this.retryOperation(func() (e error) { - hasFurtherRange, expectedRangeSize, e = this.applier.CalculateNextIterationRangeEndValues() - return e - }); err != nil { - return terminateRowIteration(err) - } - if !hasFurtherRange { - atomic.StoreInt64(&hasNoFurtherRangeFlag, 1) - return terminateRowIteration(nil) - } // Copy task: applyCopyRowsFunc := func() error { + if atomic.LoadInt64(&this.rowCopyCompleteFlag) == 1 || atomic.LoadInt64(&hasNoFurtherRangeFlag) == 1 { + // Done. + // There's another such check down the line + return nil + } + + // When hasFurtherRange is false, original table might be write locked and CalculateNextIterationRangeEndValues would hangs forever + + hasFurtherRange := false + // TODO: figure out how to rewrite this double retry? + if err := this.retryOperation(func() (e error) { + hasFurtherRange, e = this.applier.CalculateNextIterationRangeEndValues() + return e + }); err != nil { + return terminateRowIteration(err) + } + if !hasFurtherRange { + atomic.StoreInt64(&hasNoFurtherRangeFlag, 1) + return terminateRowIteration(nil) + } if atomic.LoadInt64(&this.rowCopyCompleteFlag) == 1 { // No need for more writes. // This is the de-facto place where we avoid writing in the event of completed cut-over. @@ -1286,7 +1298,7 @@ func (this *Migrator) iterateChunks() error { atomic.AddInt64(&this.migrationContext.Iteration, 1) return nil } - if err := this.retryOperation(applyCopyRowsFunc); err != nil { + if err := this.retryBatchCopyWithHooks(applyCopyRowsFunc); err != nil { return terminateRowIteration(err) } return nil From 2bc897f06b7d7a7970bacb757e0348f211362a4c Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Tue, 18 Feb 2025 11:45:55 +0100 Subject: [PATCH 02/11] Expose the last error message to the onBatchCopyRetry hook Co-authored-by: Bastian Bartmann --- go/logic/hooks.go | 5 +++-- go/logic/migrator.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 68626c9f3..5ba988a53 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -125,8 +125,9 @@ func (this *HooksExecutor) onBeforeRowCopy() error { return this.executeHooks(onBeforeRowCopy) } -func (this *HooksExecutor) onBatchCopyRetry() error { - return this.executeHooks(onBatchCopyRetry) +func (this *HooksExecutor) onBatchCopyRetry(errorMessage string) error { + v := fmt.Sprintf("GH_OST_LAST_BATCH_COPY_ERROR=%s", errorMessage) + return this.executeHooks(onBatchCopyRetry, v) } func (this *HooksExecutor) onRowCopyComplete() error { diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 5b9c4dd24..540523454 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -133,7 +133,7 @@ func (this *Migrator) sleepWhileTrue(operation func() (bool, error)) error { func (this *Migrator) retryBatchCopyWithHooks(operation func() error, notFatalHint ...bool) (err error) { wrappedOperation := func() error { if err := operation(); err != nil { - this.hooksExecutor.onBatchCopyRetry() + this.hooksExecutor.onBatchCopyRetry(err.Error()) return err } return nil From 3f8fad4a9ac8b281dbd51dab31749923e89a4230 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Fri, 21 Feb 2025 15:22:54 +0100 Subject: [PATCH 03/11] Remove double retries CalculateNextIterationRangeEndValues needs to be recomputed on every retry in case of configuration (e.g. chunk-size) changes were made by onBatchCopyRetry hooks. --- go/logic/migrator.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 540523454..72c0ad830 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1253,14 +1253,9 @@ func (this *Migrator) iterateChunks() error { } // When hasFurtherRange is false, original table might be write locked and CalculateNextIterationRangeEndValues would hangs forever - - hasFurtherRange := false - // TODO: figure out how to rewrite this double retry? - if err := this.retryOperation(func() (e error) { - hasFurtherRange, e = this.applier.CalculateNextIterationRangeEndValues() - return e - }); err != nil { - return terminateRowIteration(err) + hasFurtherRange, expectedRangeSize, err := this.applier.CalculateNextIterationRangeEndValues() + if err != nil { + return err // wrapping call will retry } if !hasFurtherRange { atomic.StoreInt64(&hasNoFurtherRangeFlag, 1) From 4e12b9add7ad89dbac6441152c42dfc567e501be Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Fri, 21 Feb 2025 15:26:08 +0100 Subject: [PATCH 04/11] include dev.yml (temp for Shopify) --- dev.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 dev.yml diff --git a/dev.yml b/dev.yml new file mode 100644 index 000000000..4c28ff349 --- /dev/null +++ b/dev.yml @@ -0,0 +1,21 @@ +name: gh-ost + +env: + TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: /var/run/docker.sock + TESTCONTAINERS_RYUK_DISABLED: "true" + +up: + - go: + version: "1.22.12" + - podman + - custom: + name: Go Dependencies + met?: go mod download + meet: echo 'go mod failed to download dependencies'; false + +commands: + test: + desc: Run all the tests. + run: | + export DOCKER_HOST=unix://$(podman machine inspect --format '{{.ConnectionInfo.PodmanSocket.Path}}') + script/test From 2c42692ee545b105cbb6c5e5780fdf7b3489d000 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Tue, 11 Mar 2025 16:11:14 +0100 Subject: [PATCH 05/11] Update doc/hooks.md --- doc/hooks.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/hooks.md b/doc/hooks.md index c1fe59453..08a450358 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -49,6 +49,7 @@ The full list of supported hooks is best found in code: [hooks.go](https://githu - `gh-ost-on-before-cut-over` - `gh-ost-on-success` - `gh-ost-on-failure` +- `gh-ost-on-batch-copy-retry` ### Context @@ -81,6 +82,7 @@ The following variable are available on particular hooks: - `GH_OST_COMMAND` is only available in `gh-ost-on-interactive-command` - `GH_OST_STATUS` is only available in `gh-ost-on-status` +- `GH_OST_LAST_BATCH_COPY_ERROR` is only available in `gh-ost-on-batch-copy-retry` ### Examples From 9e0119e58dd452164404aa9281e4dc3a32626d2a Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Wed, 26 Mar 2025 16:40:30 +0100 Subject: [PATCH 06/11] Remove dev.yml --- dev.yml | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 dev.yml diff --git a/dev.yml b/dev.yml deleted file mode 100644 index 4c28ff349..000000000 --- a/dev.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: gh-ost - -env: - TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: /var/run/docker.sock - TESTCONTAINERS_RYUK_DISABLED: "true" - -up: - - go: - version: "1.22.12" - - podman - - custom: - name: Go Dependencies - met?: go mod download - meet: echo 'go mod failed to download dependencies'; false - -commands: - test: - desc: Run all the tests. - run: | - export DOCKER_HOST=unix://$(podman machine inspect --format '{{.ConnectionInfo.PodmanSocket.Path}}') - script/test From 96f1a2881b2009b3f64ef3225148b2f566cd35a9 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Thu, 10 Apr 2025 12:59:58 +0200 Subject: [PATCH 07/11] Fix retry issue where MigrationIterationRangeMinValues advances before insert completes - extract MigrationContext.SetNextIterationRangeValues outside of applyCopyRowsFunc, so that it doesn't run on retries - add an integration test for Migrator with retry hooks Co-authored-by: Bastian Bartmann --- go/base/context.go | 7 +++ go/logic/applier.go | 4 -- go/logic/applier_test.go | 1 + go/logic/migrator.go | 1 + go/logic/migrator_test.go | 125 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 4 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index ac077076f..de65ceca5 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -584,6 +584,13 @@ func (this *MigrationContext) GetIteration() int64 { return atomic.LoadInt64(&this.Iteration) } +func (this *MigrationContext) SetNextIterationRangeValues() { + this.MigrationIterationRangeMinValues = this.MigrationIterationRangeMaxValues + if this.MigrationIterationRangeMinValues == nil { + this.MigrationIterationRangeMinValues = this.MigrationRangeMinValues + } +} + func (this *MigrationContext) MarkPointOfInterest() int64 { this.pointOfInterestTimeMutex.Lock() defer this.pointOfInterestTimeMutex.Unlock() diff --git a/go/logic/applier.go b/go/logic/applier.go index 0491aae8d..ba1fca2e8 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -664,10 +664,6 @@ func (this *Applier) ReadMigrationRangeValues() error { // no further chunk to work through, i.e. we're past the last chunk and are done with // iterating the range (and this done with copying row chunks) func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange bool, expectedRowCount int64, err error) { - this.migrationContext.MigrationIterationRangeMinValues = this.migrationContext.MigrationIterationRangeMaxValues - if this.migrationContext.MigrationIterationRangeMinValues == nil { - this.migrationContext.MigrationIterationRangeMinValues = this.migrationContext.MigrationRangeMinValues - } for i := 0; i < 2; i++ { buildFunc := sql.BuildUniqueKeyRangeEndPreparedQueryViaOffset if i == 1 { diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index d5fa06949..b1baf75e3 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -562,6 +562,7 @@ func (suite *ApplierTestSuite) TestPanicOnWarningsInApplyIterationInsertQuerySuc err = applier.ReadMigrationRangeValues() suite.Require().NoError(err) + migrationContext.SetNextIterationRangeValues() hasFurtherRange, expectedRangeSize, err := applier.CalculateNextIterationRangeEndValues() suite.Require().NoError(err) suite.Require().True(hasFurtherRange) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 72c0ad830..f77d9d9ff 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1244,6 +1244,7 @@ func (this *Migrator) iterateChunks() error { return nil } copyRowsFunc := func() error { + this.migrationContext.SetNextIterationRangeValues() // Copy task: applyCopyRowsFunc := func() error { if atomic.LoadInt64(&this.rowCopyCompleteFlag) == 1 || atomic.LoadInt64(&hasNoFurtherRangeFlag) == 1 { diff --git a/go/logic/migrator_test.go b/go/logic/migrator_test.go index 813909208..e0e1d34b7 100644 --- a/go/logic/migrator_test.go +++ b/go/logic/migrator_test.go @@ -6,9 +6,12 @@ package logic import ( + "bytes" "context" gosql "database/sql" "errors" + "fmt" + "io" "os" "path/filepath" "runtime" @@ -316,6 +319,8 @@ func (suite *MigratorTestSuite) SetupTest() { _, err := suite.db.ExecContext(ctx, "CREATE DATABASE test") suite.Require().NoError(err) + + os.Remove("/tmp/gh-ost.sock") } func (suite *MigratorTestSuite) TearDownTest() { @@ -379,6 +384,126 @@ func (suite *MigratorTestSuite) TestFoo() { suite.Require().Equal("_testing_del", tableName) } +func (suite *MigratorTestSuite) TestRetryBatchCopyWithHooks() { + ctx := context.Background() + + _, err := suite.db.ExecContext(ctx, "CREATE TABLE test.test_retry_batch (id INT PRIMARY KEY AUTO_INCREMENT, name TEXT)") + suite.Require().NoError(err) + + const initStride = 1000 + const totalBatches = 3 + for i := 0; i < totalBatches; i++ { + dataSize := 50 * i + for j := 0; j < initStride; j++ { + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("INSERT INTO test.test_retry_batch (name) VALUES ('%s')", strings.Repeat("a", dataSize))) + suite.Require().NoError(err) + } + } + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("SET GLOBAL max_binlog_cache_size = %d", 1024*8)) + suite.Require().NoError(err) + defer func() { + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("SET GLOBAL max_binlog_cache_size = %d", 1024*1024*1024)) + suite.Require().NoError(err) + }() + + tmpDir, err := os.MkdirTemp("", "gh-ost-hooks") + suite.Require().NoError(err) + defer os.RemoveAll(tmpDir) + + hookScript := filepath.Join(tmpDir, "gh-ost-on-batch-copy-retry") + hookContent := `#!/bin/bash +# Mock hook that reduces chunk size on binlog cache error +ERROR_MSG="$GH_OST_LAST_BATCH_COPY_ERROR" +SOCKET_PATH="/tmp/gh-ost.sock" + +if ! [[ "$ERROR_MSG" =~ "max_binlog_cache_size" ]]; then + echo "Nothing to do for error: $ERROR_MSG" + exit 0 +fi + +CHUNK_SIZE=$(echo "chunk-size=?" | nc -U $SOCKET_PATH | tr -d '\n') + +MIN_CHUNK_SIZE=10 +NEW_CHUNK_SIZE=$(( CHUNK_SIZE * 8 / 10 )) +if [ $NEW_CHUNK_SIZE -lt $MIN_CHUNK_SIZE ]; then + NEW_CHUNK_SIZE=$MIN_CHUNK_SIZE +fi + +if [ $CHUNK_SIZE -eq $NEW_CHUNK_SIZE ]; then + echo "Chunk size unchanged: $CHUNK_SIZE" + exit 0 +fi + +echo "[gh-ost-on-batch-copy-retry]: Changing chunk size from $CHUNK_SIZE to $NEW_CHUNK_SIZE" +echo "chunk-size=$NEW_CHUNK_SIZE" | nc -U $SOCKET_PATH +echo "[gh-ost-on-batch-copy-retry]: Done, exiting..." +` + err = os.WriteFile(hookScript, []byte(hookContent), 0755) + suite.Require().NoError(err) + + origStdout := os.Stdout + origStderr := os.Stderr + + rOut, wOut, _ := os.Pipe() + rErr, wErr, _ := os.Pipe() + os.Stdout = wOut + os.Stderr = wErr + + connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := base.NewMigrationContext() + migrationContext.AllowedRunningOnMaster = true + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.InspectorConnectionConfig = connectionConfig + migrationContext.DatabaseName = "test" + migrationContext.SkipPortValidation = true + migrationContext.OriginalTableName = "test_retry_batch" + migrationContext.SetConnectionConfig("innodb") + migrationContext.AlterStatementOptions = "MODIFY name LONGTEXT, ENGINE=InnoDB" + migrationContext.ReplicaServerId = 99999 + migrationContext.HeartbeatIntervalMilliseconds = 100 + migrationContext.ThrottleHTTPIntervalMillis = 100 + migrationContext.ThrottleHTTPTimeoutMillis = 1000 + migrationContext.HooksPath = tmpDir + migrationContext.ChunkSize = 1000 + migrationContext.SetDefaultNumRetries(10) + migrationContext.ServeSocketFile = "/tmp/gh-ost.sock" + + migrator := NewMigrator(migrationContext, "0.0.0") + + err = migrator.Migrate() + suite.Require().NoError(err) + + wOut.Close() + wErr.Close() + os.Stdout = origStdout + os.Stderr = origStderr + + var bufOut, bufErr bytes.Buffer + io.Copy(&bufOut, rOut) + io.Copy(&bufErr, rErr) + + outStr := bufOut.String() + errStr := bufErr.String() + + suite.Assert().Contains(outStr, "chunk-size: 1000") + suite.Assert().Contains(errStr, "[gh-ost-on-batch-copy-retry]: Changing chunk size from 1000 to 800") + suite.Assert().Contains(outStr, "chunk-size: 800") + + suite.Assert().Contains(errStr, "[gh-ost-on-batch-copy-retry]: Changing chunk size from 800 to 640") + suite.Assert().Contains(outStr, "chunk-size: 640") + + suite.Assert().Contains(errStr, "[gh-ost-on-batch-copy-retry]: Changing chunk size from 640 to 512") + suite.Assert().Contains(outStr, "chunk-size: 512") + + var count int + err = suite.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM test.test_retry_batch").Scan(&count) + suite.Require().NoError(err) + suite.Assert().Equal(3000, count) +} + func TestMigratorRetry(t *testing.T) { oldRetrySleepFn := RetrySleepFn defer func() { RetrySleepFn = oldRetrySleepFn }() From a48cae148228e9799caf7b43e30c782e2eac5302 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Thu, 10 Apr 2025 13:07:58 +0200 Subject: [PATCH 08/11] Add localtest that expects gh-ost to fail on exhausted retries --- localtests/copy-retries-exhausted/after.sql | 1 + localtests/copy-retries-exhausted/before.sql | 1 + localtests/copy-retries-exhausted/create.sql | 12 ++++++++++++ localtests/copy-retries-exhausted/expect_failure | 1 + localtests/copy-retries-exhausted/extra_args | 1 + localtests/test.sh | 11 +++++++++++ 6 files changed, 27 insertions(+) create mode 100644 localtests/copy-retries-exhausted/after.sql create mode 100644 localtests/copy-retries-exhausted/before.sql create mode 100644 localtests/copy-retries-exhausted/create.sql create mode 100644 localtests/copy-retries-exhausted/expect_failure create mode 100644 localtests/copy-retries-exhausted/extra_args diff --git a/localtests/copy-retries-exhausted/after.sql b/localtests/copy-retries-exhausted/after.sql new file mode 100644 index 000000000..c3d55e5b9 --- /dev/null +++ b/localtests/copy-retries-exhausted/after.sql @@ -0,0 +1 @@ +set global max_binlog_cache_size = 1073741824; -- 1GB diff --git a/localtests/copy-retries-exhausted/before.sql b/localtests/copy-retries-exhausted/before.sql new file mode 100644 index 000000000..a3570171a --- /dev/null +++ b/localtests/copy-retries-exhausted/before.sql @@ -0,0 +1 @@ +set global max_binlog_cache_size = 1024; diff --git a/localtests/copy-retries-exhausted/create.sql b/localtests/copy-retries-exhausted/create.sql new file mode 100644 index 000000000..4e37938ec --- /dev/null +++ b/localtests/copy-retries-exhausted/create.sql @@ -0,0 +1,12 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + name mediumtext not null, + primary key (id) +) auto_increment=1; + +insert into gh_ost_test (name) +select repeat('a', 1500) +from information_schema.columns +cross join information_schema.tables +limit 1000; diff --git a/localtests/copy-retries-exhausted/expect_failure b/localtests/copy-retries-exhausted/expect_failure new file mode 100644 index 000000000..cd6a516ec --- /dev/null +++ b/localtests/copy-retries-exhausted/expect_failure @@ -0,0 +1 @@ +Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage diff --git a/localtests/copy-retries-exhausted/extra_args b/localtests/copy-retries-exhausted/extra_args new file mode 100644 index 000000000..e4f8a0104 --- /dev/null +++ b/localtests/copy-retries-exhausted/extra_args @@ -0,0 +1 @@ +--alter "modify column name mediumtext" --default-retries=1 --chunk-size=1000 diff --git a/localtests/test.sh b/localtests/test.sh index e467c113b..3c3d1baf5 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -142,6 +142,12 @@ test_single() { fi gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/create.sql + + if [ -f $tests_path/$test_name/before.sql ]; then + gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/before.sql + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test < $tests_path/$test_name/before.sql + fi + test_create_result=$? if [ $test_create_result -ne 0 ] ; then @@ -208,6 +214,11 @@ test_single() { gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'" fi + if [ -f $tests_path/$test_name/after.sql ]; then + gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/after.sql + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test < $tests_path/$test_name/after.sql + fi + if [ -f $tests_path/$test_name/destroy.sql ] ; then gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/destroy.sql fi From 695a1aa6f5821f80e7ec6b35d1033fbed35e1e69 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Thu, 10 Apr 2025 13:37:55 +0200 Subject: [PATCH 09/11] Rename method --- go/base/context.go | 2 +- go/logic/applier_test.go | 2 +- go/logic/migrator.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index de65ceca5..e72d628b0 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -584,7 +584,7 @@ func (this *MigrationContext) GetIteration() int64 { return atomic.LoadInt64(&this.Iteration) } -func (this *MigrationContext) SetNextIterationRangeValues() { +func (this *MigrationContext) SetNextIterationRangeMinValues() { this.MigrationIterationRangeMinValues = this.MigrationIterationRangeMaxValues if this.MigrationIterationRangeMinValues == nil { this.MigrationIterationRangeMinValues = this.MigrationRangeMinValues diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index b1baf75e3..d1d7ce329 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -562,7 +562,7 @@ func (suite *ApplierTestSuite) TestPanicOnWarningsInApplyIterationInsertQuerySuc err = applier.ReadMigrationRangeValues() suite.Require().NoError(err) - migrationContext.SetNextIterationRangeValues() + migrationContext.SetNextIterationRangeMinValues() hasFurtherRange, expectedRangeSize, err := applier.CalculateNextIterationRangeEndValues() suite.Require().NoError(err) suite.Require().True(hasFurtherRange) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index f77d9d9ff..30ab4ca5a 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1244,7 +1244,7 @@ func (this *Migrator) iterateChunks() error { return nil } copyRowsFunc := func() error { - this.migrationContext.SetNextIterationRangeValues() + this.migrationContext.SetNextIterationRangeMinValues() // Copy task: applyCopyRowsFunc := func() error { if atomic.LoadInt64(&this.rowCopyCompleteFlag) == 1 || atomic.LoadInt64(&hasNoFurtherRangeFlag) == 1 { From 512b07228dad504954dfe477cbd4501594ce4c4b Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Wed, 6 Aug 2025 13:19:48 +0200 Subject: [PATCH 10/11] fmt and lint --- go/logic/migrator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/migrator_test.go b/go/logic/migrator_test.go index c16aa21f8..46435f1d3 100644 --- a/go/logic/migrator_test.go +++ b/go/logic/migrator_test.go @@ -10,8 +10,8 @@ import ( "context" gosql "database/sql" "errors" - "io" "fmt" + "io" "os" "path/filepath" "strings" From 24595e485f1bb579d90b5b4e17ed17dbb871d433 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Thu, 6 Nov 2025 18:47:04 +0100 Subject: [PATCH 11/11] gofmt --- go/logic/applier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 4c7579948..5a87c1560 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -824,7 +824,7 @@ func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange boo this.LastIterationRangeMaxValues = this.migrationContext.MigrationIterationRangeMaxValues.Clone() } this.LastIterationRangeMutex.Unlock() - + for i := 0; i < 2; i++ { buildFunc := sql.BuildUniqueKeyRangeEndPreparedQueryViaOffset if i == 1 {