From 7f207a291cb8b9903c08a05b0daa0a7eb90c3921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 29 May 2026 17:11:13 +0300 Subject: [PATCH 01/25] WIP MDEV-14992 BACKUP SERVER This introduces a basic driver Sql_cmd_backup, storage engine interfaces, and basic copying of InnoDB data files. On Windows, we pass a target directory name; elsewhere, we pass a target directory handle. backup_target: A structured data type to represent a directory or a stream. On Microsoft Windows, we must use directory paths because there is no variant of CopyFileEx() that would work on file handles. copy_entire_file(): A file copying service for POSIX systems. copy_file(): A sparse file-copying service for POSIX systems. backup_context: An InnoDB backup context, attached to trx->lock.backup so that context can exist between InnoDB_backup::end(), which is releasing all locks, and InnoDB_backup::fini() in the same thread, which is expected to finalize the backup without modifying files in the server data directory. fil_space_t::write_or_backup: Keep track of in-flight page writes and pending backup operation. We must not allow them concurrently, because that could lead into torn pages in the backup. fil_space_t::backup_end: The first page number that is not being backed up (by default 0, to indicate that no backup is in progress). TRX_STATE_BACKUP: A special InnoDB transaction state indicating association with BACKUP SERVER, which allows us to pass some context in trx_t from innodb_backup_end() to innodb_backup_finalize(). log_t::backup: Whether BACKUP SERVER is in progress. The purpose of this is to make BACKUP SERVER prevent the concurrent execution of SET GLOBAL innodb_log_archive=OFF or SET GLOBAL innodb_log_file_size when innodb_log_archive=OFF. log_sys.archived_checkpoint: Keep track of the earliest available checkpoint, corresponding to log_sys.archived_lsn. This reflects SET GLOBAL innodb_log_recovery_start (which is settable now), for incremental backup. buf_flush_list_space(): Check for concurrent backup before writing each page. This is inefficient, but this function may be invoked from multiple threads concurrently, and it cannot be changed easily, especially for fil_crypt_thread(). --- libmysqld/CMakeLists.txt | 1 + mysql-test/collections/buildbot_suites.bat | 1 + mysql-test/main/backup_server.result | 6 + mysql-test/main/backup_server.test | 10 + mysql-test/main/backup_server_locking.result | 17 + mysql-test/main/backup_server_locking.test | 31 + mysql-test/main/grant_backup_server.result | 27 + mysql-test/main/grant_backup_server.test | 29 + mysql-test/main/mysqld--help.result | 2 +- mysql-test/mariadb-test-run.pl | 1 + .../suite/backup/backup_innodb,debug.rdiff | 16 + .../suite/backup/backup_innodb.combinations | 4 + mysql-test/suite/backup/backup_innodb.result | 24 + mysql-test/suite/backup/backup_innodb.test | 49 ++ mysql-test/suite/backup/suite.pm | 10 + .../perfschema/r/max_program_zero.result | 2 +- .../suite/perfschema/r/ortho_iter.result | 2 +- .../perfschema/r/privilege_table_io.result | 2 +- .../r/start_server_disable_idle.result | 2 +- .../r/start_server_disable_stages.result | 2 +- .../r/start_server_disable_statements.result | 2 +- .../start_server_disable_transactions.result | 2 +- .../r/start_server_disable_waits.result | 2 +- .../perfschema/r/start_server_innodb.result | 2 +- .../r/start_server_low_index.result | 2 +- .../r/start_server_low_table_lock.result | 2 +- .../r/start_server_no_account.result | 2 +- .../r/start_server_no_cond_class.result | 2 +- .../r/start_server_no_cond_inst.result | 2 +- .../r/start_server_no_file_class.result | 2 +- .../r/start_server_no_file_inst.result | 2 +- .../perfschema/r/start_server_no_host.result | 2 +- .../perfschema/r/start_server_no_index.result | 2 +- .../perfschema/r/start_server_no_mdl.result | 2 +- .../r/start_server_no_memory_class.result | 2 +- .../r/start_server_no_mutex_class.result | 2 +- .../r/start_server_no_mutex_inst.result | 2 +- ..._server_no_prepared_stmts_instances.result | 2 +- .../r/start_server_no_rwlock_class.result | 2 +- .../r/start_server_no_rwlock_inst.result | 2 +- .../r/start_server_no_setup_actors.result | 2 +- .../r/start_server_no_setup_objects.result | 2 +- .../r/start_server_no_socket_class.result | 2 +- .../r/start_server_no_socket_inst.result | 2 +- .../r/start_server_no_stage_class.result | 2 +- .../r/start_server_no_stages_history.result | 2 +- ...start_server_no_stages_history_long.result | 2 +- .../start_server_no_statements_history.result | 2 +- ...t_server_no_statements_history_long.result | 2 +- .../r/start_server_no_table_hdl.result | 2 +- .../r/start_server_no_table_inst.result | 2 +- .../r/start_server_no_table_lock.result | 2 +- .../r/start_server_no_thread_class.result | 2 +- .../r/start_server_no_thread_inst.result | 2 +- ...tart_server_no_transactions_history.result | 2 +- ...server_no_transactions_history_long.result | 2 +- .../perfschema/r/start_server_no_user.result | 2 +- .../r/start_server_no_waits_history.result | 2 +- .../start_server_no_waits_history_long.result | 2 +- .../perfschema/r/start_server_off.result | 2 +- .../suite/perfschema/r/start_server_on.result | 2 +- .../r/start_server_variables.result | 2 +- .../r/statement_program_lost_inst.result | 2 +- .../suite/sys_vars/r/sysvars_innodb.result | 2 +- sql/CMakeLists.txt | 1 + sql/handler.h | 56 ++ sql/mysqld.cc | 1 + sql/sql_backup.cc | 299 +++++++ sql/sql_backup.h | 36 + sql/sql_backup_interface.h | 57 ++ sql/sql_command.h | 1 + sql/sql_parse.cc | 4 +- sql/sql_yacc.yy | 6 + sql/sys_vars.inl | 2 + storage/innobase/CMakeLists.txt | 2 + storage/innobase/buf/buf0flu.cc | 99 ++- storage/innobase/handler/backup_innodb.cc | 784 ++++++++++++++++++ storage/innobase/handler/backup_innodb.h | 54 ++ storage/innobase/handler/ha_innodb.cc | 69 +- storage/innobase/include/fil0fil.h | 44 + storage/innobase/include/log0log.h | 62 +- storage/innobase/include/trx0trx.h | 12 +- storage/innobase/include/trx0trx.inl | 1 + storage/innobase/include/trx0types.h | 23 +- storage/innobase/log/log0log.cc | 78 +- storage/innobase/log/log0recv.cc | 4 +- storage/innobase/os/os0file.cc | 10 +- storage/innobase/trx/trx0roll.cc | 3 + storage/innobase/trx/trx0sys.cc | 2 + storage/innobase/trx/trx0trx.cc | 7 + 90 files changed, 1901 insertions(+), 142 deletions(-) create mode 100644 mysql-test/main/backup_server.result create mode 100644 mysql-test/main/backup_server.test create mode 100644 mysql-test/main/backup_server_locking.result create mode 100644 mysql-test/main/backup_server_locking.test create mode 100644 mysql-test/main/grant_backup_server.result create mode 100644 mysql-test/main/grant_backup_server.test create mode 100644 mysql-test/suite/backup/backup_innodb,debug.rdiff create mode 100644 mysql-test/suite/backup/backup_innodb.combinations create mode 100644 mysql-test/suite/backup/backup_innodb.result create mode 100644 mysql-test/suite/backup/backup_innodb.test create mode 100644 mysql-test/suite/backup/suite.pm create mode 100644 sql/sql_backup.cc create mode 100644 sql/sql_backup.h create mode 100644 sql/sql_backup_interface.h create mode 100644 storage/innobase/handler/backup_innodb.cc create mode 100644 storage/innobase/handler/backup_innodb.h diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index 80b202aa4c03e..59f386a86edae 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -166,6 +166,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/opt_hints.cc ../sql/opt_hints.h ../sql/opt_trace_ddl_info.cc ../sql/opt_trace_ddl_info.h ../sql/sql_path.cc + ../sql/sql_backup.cc ${GEN_SOURCES} ${MYSYS_LIBWRAP_SOURCE} ) diff --git a/mysql-test/collections/buildbot_suites.bat b/mysql-test/collections/buildbot_suites.bat index 053057872bf40..129a2f67d79de 100644 --- a/mysql-test/collections/buildbot_suites.bat +++ b/mysql-test/collections/buildbot_suites.bat @@ -6,6 +6,7 @@ innodb,^ versioning,^ plugins,^ mariabackup,^ +backup,^ roles,^ auth_gssapi,^ mysql_sha2,^ diff --git a/mysql-test/main/backup_server.result b/mysql-test/main/backup_server.result new file mode 100644 index 0000000000000..7a064d7e5fe11 --- /dev/null +++ b/mysql-test/main/backup_server.result @@ -0,0 +1,6 @@ +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; diff --git a/mysql-test/main/backup_server.test b/mysql-test/main/backup_server.test new file mode 100644 index 0000000000000..50cf326d39838 --- /dev/null +++ b/mysql-test/main/backup_server.test @@ -0,0 +1,10 @@ +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory diff --git a/mysql-test/main/backup_server_locking.result b/mysql-test/main/backup_server_locking.result new file mode 100644 index 0000000000000..6e49a09e3e873 --- /dev/null +++ b/mysql-test/main/backup_server_locking.result @@ -0,0 +1,17 @@ +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +BACKUP STAGE START; +connect backup,localhost,root; +SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR 70100: Query was interrupted: execution time limit 0.1 sec exceeded +connection default; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't execute the command as you have a BACKUP STAGE active +BACKUP STAGE END; +connection backup; +SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +disconnect backup; +connection default; diff --git a/mysql-test/main/backup_server_locking.test b/mysql-test/main/backup_server_locking.test new file mode 100644 index 0000000000000..4e769b61b2f20 --- /dev/null +++ b/mysql-test/main/backup_server_locking.test @@ -0,0 +1,31 @@ +--source include/not_embedded.inc +--source include/count_sessions.inc + +--mkdir $MYSQLTEST_VARDIR/some_directory +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +BACKUP STAGE START; +--connect (backup,localhost,root) +--error ER_STATEMENT_TIMEOUT +evalp SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +--connection default + +--error ER_BACKUP_LOCK_IS_ACTIVE +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +BACKUP STAGE END; +--connection backup +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--disconnect backup +--connection default + +--rmdir $MYSQLTEST_VARDIR/some_directory + +--source include/wait_until_count_sessions.inc diff --git a/mysql-test/main/grant_backup_server.result b/mysql-test/main/grant_backup_server.result new file mode 100644 index 0000000000000..a4d85292dee45 --- /dev/null +++ b/mysql-test/main/grant_backup_server.result @@ -0,0 +1,27 @@ +CREATE USER user1@localhost IDENTIFIED BY ''; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON test.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT RELOAD ON test.* TO user1@localhost; +ERROR HY000: Incorrect usage of DB GRANT and GLOBAL PRIVILEGES +GRANT RELOAD ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the SELECT privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +disconnect con1; +connection default; +DROP USER user1@localhost; diff --git a/mysql-test/main/grant_backup_server.test b/mysql-test/main/grant_backup_server.test new file mode 100644 index 0000000000000..726abb19908bb --- /dev/null +++ b/mysql-test/main/grant_backup_server.test @@ -0,0 +1,29 @@ +--source include/not_embedded.inc +CREATE USER user1@localhost IDENTIFIED BY ''; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON test.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +--error ER_WRONG_USAGE +GRANT RELOAD ON test.* TO user1@localhost; +GRANT RELOAD ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +--disconnect con1 +--connection default +DROP USER user1@localhost; diff --git a/mysql-test/main/mysqld--help.result b/mysql-test/main/mysqld--help.result index d87ce7f156a06..268efe5c8b960 100644 --- a/mysql-test/main/mysqld--help.result +++ b/mysql-test/main/mysqld--help.result @@ -2053,7 +2053,7 @@ performance-schema-max-socket-classes 10 performance-schema-max-socket-instances -1 performance-schema-max-sql-text-length 1024 performance-schema-max-stage-classes 170 -performance-schema-max-statement-classes 227 +performance-schema-max-statement-classes 228 performance-schema-max-statement-stack 10 performance-schema-max-table-handles -1 performance-schema-max-table-instances -1 diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index 7bea206fb2ff5..1182109f7119b 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -180,6 +180,7 @@ END main- archive- atomic- + backup- binlog- binlog_encryption- binlog_in_engine- diff --git a/mysql-test/suite/backup/backup_innodb,debug.rdiff b/mysql-test/suite/backup/backup_innodb,debug.rdiff new file mode 100644 index 0000000000000..02d25801b75a7 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb,debug.rdiff @@ -0,0 +1,16 @@ +--- backup_innodb.result ++++ backup_innodb,debug.result +@@ -10,7 +10,13 @@ + BEGIN; + DELETE FROM t; + connect backup,localhost,root; ++SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; + BACKUP SERVER TO 'MYSQLTEST_VARDIR/some_directory'; ++connection default; ++SET DEBUG_SYNC='now WAIT_FOR start'; ++INSERT INTO t(a) SELECT * FROM seq_1_to_30000; ++SET DEBUG_SYNC='now SIGNAL resume'; ++connection backup; + disconnect backup; + connection default; + ROLLBACK; diff --git a/mysql-test/suite/backup/backup_innodb.combinations b/mysql-test/suite/backup/backup_innodb.combinations new file mode 100644 index 0000000000000..7fd419bba7692 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.combinations @@ -0,0 +1,4 @@ +[archived] +innodb_log_archive=ON +[circular] +innodb_log_archive=OFF diff --git a/mysql-test/suite/backup/backup_innodb.result b/mysql-test/suite/backup/backup_innodb.result new file mode 100644 index 0000000000000..e1e856e93bae0 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.result @@ -0,0 +1,24 @@ +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ROLLBACK; +SELECT * FROM t; +a b +1 +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +BACKUP SERVER TO 'MYSQLTEST_VARDIR/some_directory'; +disconnect backup; +connection default; +ROLLBACK; +SELECT * FROM t; +a b +1 +# restart +SELECT * FROM t; +a b +1 +DROP TABLE t; diff --git a/mysql-test/suite/backup/backup_innodb.test b/mysql-test/suite/backup/backup_innodb.test new file mode 100644 index 0000000000000..5f7bde34a31dd --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.test @@ -0,0 +1,49 @@ +--source include/have_sequence.inc +--source include/have_innodb.inc +--source include/maybe_debug.inc + +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; + +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory +ROLLBACK; +# BACKUP SERVER will implicitly commit the current transaction +SELECT * FROM t; + +BEGIN; +DELETE FROM t; + +--connect backup,localhost,root +if ($have_debug) { +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +send_eval BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--connection default +SET DEBUG_SYNC='now WAIT_FOR start'; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +SET DEBUG_SYNC='now SIGNAL resume'; +--connection backup +--reap +} +if (!$have_debug) { +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +eval BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +} + +--disconnect backup +--connection default +ROLLBACK; +SELECT * FROM t; + +let $datadir=`SELECT @@datadir`; +--source include/shutdown_mysqld.inc +#--rmdir $datadir +#--move_file $MYSQLTEST_VARDIR/some_directory $datadir +--rmdir $MYSQLTEST_VARDIR/some_directory +--source include/start_mysqld.inc + +SELECT * FROM t; +DROP TABLE t; diff --git a/mysql-test/suite/backup/suite.pm b/mysql-test/suite/backup/suite.pm new file mode 100644 index 0000000000000..e24cb5b5185b1 --- /dev/null +++ b/mysql-test/suite/backup/suite.pm @@ -0,0 +1,10 @@ +package My::Suite::Backup; + +@ISA = qw(My::Suite); +use My::Find; +use File::Basename; +use strict; + +return "Not run for embedded server" if $::opt_embedded_server; + +bless { }; diff --git a/mysql-test/suite/perfschema/r/max_program_zero.result b/mysql-test/suite/perfschema/r/max_program_zero.result index 047643e06988d..a0e486b2af9c0 100644 --- a/mysql-test/suite/perfschema/r/max_program_zero.result +++ b/mysql-test/suite/perfschema/r/max_program_zero.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 1 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/ortho_iter.result b/mysql-test/suite/perfschema/r/ortho_iter.result index 56c22c8453d8e..589704f4056de 100644 --- a/mysql-test/suite/perfschema/r/ortho_iter.result +++ b/mysql-test/suite/perfschema/r/ortho_iter.result @@ -251,7 +251,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/privilege_table_io.result b/mysql-test/suite/perfschema/r/privilege_table_io.result index 0c82be9f05810..b518052613308 100644 --- a/mysql-test/suite/perfschema/r/privilege_table_io.result +++ b/mysql-test/suite/perfschema/r/privilege_table_io.result @@ -57,7 +57,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_idle.result b/mysql-test/suite/perfschema/r/start_server_disable_idle.result index d0665e3bf4c65..12b956d90769c 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_idle.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_idle.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_stages.result b/mysql-test/suite/perfschema/r/start_server_disable_stages.result index 2ef68328144ff..30ce9d12e56a0 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_stages.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_stages.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_statements.result b/mysql-test/suite/perfschema/r/start_server_disable_statements.result index 0ece2a0c52ed1..4bacdbdfedf68 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_statements.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_statements.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result index ededc09aac95d..3a6831eb2c7ff 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_waits.result b/mysql-test/suite/perfschema/r/start_server_disable_waits.result index 23db9362161e4..a864576dfa979 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_waits.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_waits.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_innodb.result b/mysql-test/suite/perfschema/r/start_server_innodb.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_innodb.result +++ b/mysql-test/suite/perfschema/r/start_server_innodb.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_index.result b/mysql-test/suite/perfschema/r/start_server_low_index.result index 11cade0a2132f..dbd36d2eaa92d 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_index.result +++ b/mysql-test/suite/perfschema/r/start_server_low_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result index 484550095202e..fab64f49d45e6 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_account.result b/mysql-test/suite/perfschema/r/start_server_no_account.result index aab8d3eba9caa..e2cccdba19b43 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_account.result +++ b/mysql-test/suite/perfschema/r/start_server_no_account.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result index 4dfdab9de9f30..44b114013ace2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result index a8d0cbeca3855..27ecb59a40f17 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_class.result b/mysql-test/suite/perfschema/r/start_server_no_file_class.result index fcc01880a7107..5189d36618b05 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result index c56201e7d0a80..533c02383c7b8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_host.result b/mysql-test/suite/perfschema/r/start_server_no_host.result index 662beb3b88a49..e328ea3d3c26a 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_host.result +++ b/mysql-test/suite/perfschema/r/start_server_no_host.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_index.result b/mysql-test/suite/perfschema/r/start_server_no_index.result index ccff0cb113faa..686f430cdc440 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_index.result +++ b/mysql-test/suite/perfschema/r/start_server_no_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mdl.result b/mysql-test/suite/perfschema/r/start_server_no_mdl.result index ebe64409deb0c..c2b6dc3d9eb74 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result index 01a217c534bfd..2a4cdcf8ddd37 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result index 1b3efda5210a9..9b77c7b897c47 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result index 599498915f3f1..3a62653efc18f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result index 73ac1acb9f145..7dd5842809135 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result +++ b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result index 04aa037b960e3..3598c722b9453 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result index e8156711eb3f2..7a60805d2d03f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result index 2d17bd6f49203..76da2883ba6b2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result index 16afee28ee70b..58ac763394a96 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result index 9de0006aa7abc..5a86d51a06231 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 0 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result index bcef0e5c01b05..75ab84f4b3cb8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 0 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result index 1dda39dc79e92..bb750ebd34a26 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 0 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result index 95584521eceb2..6cf3f740fb1be 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result index da6c54b6bbafa..1f2716410a9bc 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result index 09a9a544f5b13..f1078542f69ef 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result index 5ce9874799e8b..a6fc523df2c60 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result index a170fe097fd23..304732cb37927 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 0 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result index 3f009de021d1a..3ef43495d0f22 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 0 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result index db4fe3413106b..20cf1a531ca51 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result index b1b6e614c43ac..e8eb4589fce56 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result index e540f3ce78cbd..4dbf292fc3c10 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result index 80da238ba0ed7..09168c68f9fe8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result index f5d32f0100968..9ec63e991deda 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_user.result b/mysql-test/suite/perfschema/r/start_server_no_user.result index cb249b4e242d3..861b7d897c9c5 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_user.result +++ b/mysql-test/suite/perfschema/r/start_server_no_user.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result index c419aad2995f3..710cef232da58 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result index 0b2bf39ff874f..df4351a8f358f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_off.result b/mysql-test/suite/perfschema/r/start_server_off.result index 17db0395d894b..f3bf7bf7fb15e 100644 --- a/mysql-test/suite/perfschema/r/start_server_off.result +++ b/mysql-test/suite/perfschema/r/start_server_off.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_on.result b/mysql-test/suite/perfschema/r/start_server_on.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_on.result +++ b/mysql-test/suite/perfschema/r/start_server_on.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_variables.result b/mysql-test/suite/perfschema/r/start_server_variables.result index f25f9ab69d1c3..b86116e099cdd 100644 --- a/mysql-test/suite/perfschema/r/start_server_variables.result +++ b/mysql-test/suite/perfschema/r/start_server_variables.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result index 442e212557b55..3e8469b881801 100644 --- a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result +++ b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 2 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 11a75e91ae34a..d8afac2c2c63b 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1052,7 +1052,7 @@ NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 18446744073709551615 NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL -READ_ONLY YES +READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_LOG_RECOVERY_TARGET SESSION_VALUE NULL diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index e54e894e1d0fc..fd16d901d64fb 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -163,6 +163,7 @@ SET (SQL_SOURCE grant.cc sql_explain.cc sql_analyze_stmt.cc + sql_backup.cc sql_join_cache.cc create_options.cc multi_range_read.cc opt_histogram_json.cc diff --git a/sql/handler.h b/sql/handler.h index 3ab9e0bcd1a8e..e9a2f7e164e08 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -1496,6 +1496,27 @@ struct transaction_participant ulonglong (*prepare_commit_versioned)(THD *thd, ulonglong *trx_id); }; +/** BACKUP SERVER target */ +struct backup_target +{ +#ifdef _WIN32 + /** Target directory path name */ + const char *path; + union + { + /** Target pipe, if path==reinterpret_cast(-1) */ + HANDLE pipe; + /** Target socket, if path==nullptr */ + SOCKET socket; + }; +#else + /** Target file descriptor */ + int fd; + /** whether the fd is a directory handle */ + bool directory; +#endif +}; + /* handlerton is a singleton structure - one instance per storage engine - to provide access to storage engine functionality that works on the @@ -1892,9 +1913,44 @@ struct handlerton : public transaction_participant /********************************************************************* backup **********************************************************************/ + + /** BACKUP STAGE START */ void (*prepare_for_backup)(void); + /** BACKUP STAGE END */ void (*end_backup)(void); + /** + Start of BACKUP SERVER: collect all files to be backed up + @param thd current session + @param target backup target + @return error code + @retval 0 on success + */ + int (*backup_start)(THD *thd, backup_target target); + /** + Process a file that was collected in backup_start(). + @param thd current session + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int (*backup_step)(THD *thd); + /** + Finish copying and determine the logical time of the backup snapshot. + @param thd current sesssion + @param abort whether BACKUP SERVER was aborted + @return error code + @retval 0 on success + */ + int (*backup_end)(THD *thd, bool abort); + /** + Clean up after any backup_end(). + @param thd the parameter on which backup_end() was invoked + @param target backup target + @return error code + @retval 0 on success + */ + int (*backup_finalize)(THD *thd, backup_target target); + /********************************************************************** WSREP specific **********************************************************************/ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index de79500825457..111fcb0aa7505 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3543,6 +3543,7 @@ SHOW_VAR com_status_vars[]= { {"assign_to_keycache", STMT_STATUS(SQLCOM_ASSIGN_TO_KEYCACHE)}, {"backup", STMT_STATUS(SQLCOM_BACKUP)}, {"backup_lock", STMT_STATUS(SQLCOM_BACKUP_LOCK)}, + {"backup_server", STMT_STATUS(SQLCOM_BACKUP_SERVER)}, {"begin", STMT_STATUS(SQLCOM_BEGIN)}, {"binlog", STMT_STATUS(SQLCOM_BINLOG_BASE64_EVENT)}, {"call_procedure", STMT_STATUS(SQLCOM_CALL)}, diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc new file mode 100644 index 0000000000000..224345c5d6ec6 --- /dev/null +++ b/sql/sql_backup.cc @@ -0,0 +1,299 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "mdl.h" +#include "mysys_err.h" +#include "sql_class.h" +#include "sql_backup.h" +#include "sql_backup_interface.h" +#include "sql_parse.h" + +#ifndef _WIN32 +using copying_step= ssize_t(int,int,size_t,off_t*); +template +static ssize_t copy(int in_fd, int out_fd, off_t offset, off_t end) noexcept +{ + for (;;) + { + const size_t c{size_t(std::min(end - offset, INT_MAX >> 20 << 20))}; + ssize_t ret= step(in_fd, out_fd, c, &offset); + if (ret < 0) + return ret; + if (offset == end) + return 0; + if (!ret) + return -1; + } +} +# if defined __linux__ || defined __FreeBSD__ +/* Copy between files in a single (type of) file system */ +static inline ssize_t +copy_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return copy_file_range(in_fd, offset, out_fd, offset, count, 0); +} +# define cfr(src,dst,start,end) copy(src, dst, start, end) +# endif +# ifdef __linux__ +# include +/* Copy a file to a stream or to a regular file. */ +static inline ssize_t +send_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return sendfile(out_fd, in_fd, offset, count); +} +# else +# include "aligned.h" +# include +/** Copy a file using a memory mapping. +@param in_fd source file +@param out_fd destination +@param o start offset +@param end last offset (exclusive) +@return error code +@retval 0 on success +@retval 1 if a memory mapping failed */ +static ssize_t mmap_copy(int in_fd, int out_fd, off_t o, off_t end) +{ + assert(end >= o); +#if SIZEOF_SIZE_T < 8 + if (end != ssize_t(end)) + return 1; +#endif + const size_t count= size_t(end - o); + void *p= mmap(nullptr, count, PROT_READ, MAP_SHARED, in_fd, o); + if (p == MAP_FAILED) + return 1; + ssize_t ret; + size_t c{count}; + for (const char *b= static_cast(p);; b+= ret, o+= ret) + { + ret= pwrite(out_fd, b, std::min(c, size_t(INT_MAX >> 20 << 20)), o); + if (ret < 0) + break; + c-= ret; + if (!c) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + munmap(p, c); + return ret; +} + +/** Copy a file using positioned reads and writes. +@param in_fd source file +@param out_fd destination +@param o start offset +@param end last offset (exclusive) +@return error code +@retval 0 on success +@retval 1 if a memory mapping failed */ +static ssize_t pread_pwrite(int in_fd, int out_fd, off_t o, off_t end) + noexcept +{ + assert(end >= o); + constexpr size_t READ_WRITE_SIZE= 65536; + char *b= static_cast(aligned_malloc(READ_WRITE_SIZE, 4096)); + if (!b) + return -1; + ssize_t ret; + for (off_t count{end - o};; o+= ret) + { + ret= pread(in_fd, b, ssize_t(std::min(count, off_t{READ_WRITE_SIZE})), o); + if (ret > 0) + ret= pwrite(out_fd, b, ret, o); + if (ret < 0) + break; + count-= ret; + if (!count) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + aligned_free(b); + return ret; +} +# endif + +#ifdef __APPLE__ +/* The inline copy_entire_file() invokes fcopyfile() */ +#else +/** Copy a file (whole content). +@param src source file descriptor +@param dst target to append src to +@return error code (negative) +@retval 0 on success */ +extern "C" int copy_entire_file(int src, int dst) +{ + return copy_file(src, dst, 0, lseek(src, 0, SEEK_END)); +} +#endif + +/** Copy a file. +@param src source file descriptor +@param dst target to append src to +@param start first offset to copy +@param end last offset to copy (exclusive) +@return error code (negative) +@retval 0 on success */ +extern "C" int copy_file(int src, int dst, off_t start, off_t end) +{ + assert(end >= start); + ssize_t ret; +# ifdef cfr + if (!(ret= cfr(src, dst, start, end))) + return int(ret); +# ifdef __linux__ + if (errno == EOPNOTSUPP || errno == EXDEV) +# endif +# endif +# ifdef __linux__ // starting with Linux 2.6.33, we can rely on sendfile(2) + ret= (start != 0 && start != lseek(dst, start, SEEK_SET)) + ? -1 + : copy(src, dst, start, end); +# else + if ((ret= mmap_copy(src, dst, start, end)) == 1) + ret= pread_pwrite(src, dst, start, end); +# endif + assert(ret <= 0); + return int(ret); +} +#endif + +static my_bool backup_start(THD *thd, plugin_ref plugin, void *dst) noexcept +{ + handlerton *hton= plugin_hton(plugin); + if (hton->backup_start) + return hton->backup_start(thd, *static_cast(dst)); + return false; +} + +static my_bool backup_end(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + handlerton *hton= plugin_hton(plugin); + if (hton->backup_end) + return hton->backup_end(thd, arg != nullptr); + return false; +} + +static my_bool backup_step(THD *thd, plugin_ref plugin, void *) noexcept +{ + handlerton *hton= plugin_hton(plugin); + int res= 0; + if (hton->backup_step) + while ((res= hton->backup_step(thd))) + if (res < 0) + break; + return res != 0; +} + +static my_bool backup_finalize(THD *thd, plugin_ref plugin, void *dst) noexcept +{ + handlerton *hton= plugin_hton(plugin); + if (hton->backup_finalize) + return hton->backup_finalize(thd, *static_cast(dst)); + return 0; +} + +bool Sql_cmd_backup::execute(THD *thd) +{ + if (check_global_access(thd, RELOAD_ACL) || + check_global_access(thd, SELECT_ACL) || + error_if_data_home_dir(target.str, "BACKUP SERVER TO")) + return true; + + if (thd->current_backup_stage != BACKUP_FINISHED) + { + my_error(ER_BACKUP_LOCK_IS_ACTIVE, MYF(0)); + return true; + } + + /* Block concurrent BACKUP SERVER and BACKUP STAGE */ + MDL_request mdl_request; + MDL_REQUEST_INIT(&mdl_request, MDL_key::BACKUP, "", "", MDL_BACKUP_START, + MDL_EXPLICIT); + + if (thd->mdl_context.acquire_lock(&mdl_request, + thd->variables.lock_wait_timeout)) + return true; + + if (my_mkdir(target.str, 0755, MYF(MY_WME))) + { +#ifndef _WIN32 + err_exit: +#endif + thd->mdl_context.release_lock(mdl_request.ticket); + return true; + } + +#ifdef _WIN32 + backup_target dir{target.str, INVALID_HANDLE_VALUE}; +#else + backup_target dir{open(target.str, O_DIRECTORY), true}; + if (dir.fd < 0) + { + my_error(EE_CANT_MKDIR, MYF(ME_BELL), target.str, errno); + goto err_exit; + } +#endif + + bool fail= plugin_foreach_with_mask(thd, backup_start, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, &dir); + + /* The backup_step may be invoked in multiple concurrent threads. + At the time backup_end is invoked, all backup_step will have to complete. */ + if (!fail) + fail= plugin_foreach_with_mask(thd, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, nullptr); + + fail= + thd->mdl_context.upgrade_shared_lock(mdl_request.ticket, + MDL_BACKUP_WAIT_COMMIT, + thd->variables.lock_wait_timeout) || + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + reinterpret_cast(fail)) || fail; + + /* The final part must not interfere with the use of the server datadir. + Release the locks. */ + thd->mdl_context.release_lock(mdl_request.ticket); + fail= plugin_foreach_with_mask(thd, backup_finalize, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, &dir) || + fail; +#ifndef _WIN32 + close(dir.fd); +#endif + + if (!fail) + my_ok(thd); + return fail; +} diff --git a/sql/sql_backup.h b/sql/sql_backup.h new file mode 100644 index 0000000000000..9aba2404dac58 --- /dev/null +++ b/sql/sql_backup.h @@ -0,0 +1,36 @@ +/***************************************************************************** +Copyright (c) 2026 MariaDB plc. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +#pragma once + +/** BACKUP SERVER */ +class Sql_cmd_backup : public Sql_cmd +{ + /** target directory */ + const LEX_CSTRING target; + +public: + explicit Sql_cmd_backup(LEX_CSTRING target) : target(target) {} + ~Sql_cmd_backup() = default; + + bool execute(THD *thd) override; + + enum_sql_command sql_command_code() const override + { + return SQLCOM_BACKUP_SERVER; + } +}; diff --git a/sql/sql_backup_interface.h b/sql/sql_backup_interface.h new file mode 100644 index 0000000000000..a86a16ab505fc --- /dev/null +++ b/sql/sql_backup_interface.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#ifdef _WIN32 +/* You have to use CopyFileEx() and friends manually */ +#else +# if defined __APPLE__ +/* You should invoke fclonefileat(2) manually before attempting +copy_entire_file() or copy_file() */ +# include +# include +# include +/** Copy an entire file. +@param src source file descriptor +@param dst target to append src to +@return error code (negative) +@retval 0 on success */ +inline int copy_entire_file(int src, int dst) +{ + return fcopyfile(src, dst, NULL, COPYFILE_ALL | COPYFILE_CLONE); +} +# else +# ifdef __cplusplus +extern "C" +# endif +/** Copy an entire file. +@param src source file descriptor +@param dst target to append src to +@return error code (negative) +@retval 0 on success */ +int copy_entire_file(int src, int dst); +# endif + +# ifdef __cplusplus +extern "C" +# endif +/** Copy a file. +@param src source file descriptor +@param dst target to append src to +@param start first offset to copy +@param end last offset to copy (exclusive) +@return error code (negative) +@retval 0 on success */ +int copy_file(int src, int dst, off_t start, off_t end); +#endif diff --git a/sql/sql_command.h b/sql/sql_command.h index 9c9166706a034..b8903399711f0 100644 --- a/sql/sql_command.h +++ b/sql/sql_command.h @@ -103,6 +103,7 @@ enum enum_sql_command { SQLCOM_SHOW_PACKAGE_BODY_CODE, SQLCOM_BACKUP, SQLCOM_BACKUP_LOCK, SQLCOM_SHOW_CREATE_SERVER, + SQLCOM_BACKUP_SERVER, /* When a command is added here, be sure it's also added in mysqld.cc diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index bcd6de564f550..b549017622332 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -781,6 +781,7 @@ void init_update_queries(void) sql_command_flags[SQLCOM_DROP_SERVER]|= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP]= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP_LOCK]= CF_AUTO_COMMIT_TRANS; + sql_command_flags[SQLCOM_BACKUP_SERVER]= CF_AUTO_COMMIT_TRANS; /* The following statements can deal with temporary tables, @@ -5899,6 +5900,7 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt) case SQLCOM_CALL: case SQLCOM_REVOKE: case SQLCOM_GRANT: + case SQLCOM_BACKUP_SERVER: if (thd->variables.option_bits & OPTION_IF_EXISTS) lex->create_info.set(DDL_options_st::OPT_IF_EXISTS); DBUG_ASSERT(lex->m_sql_cmd != NULL); @@ -10254,7 +10256,7 @@ int test_if_data_home_dir(const char *dir) if (!dir) DBUG_RETURN(0); - (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH); + (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH|MY_RESOLVE_SYMLINKS); DBUG_RETURN(path_starts_from_data_home_dir(path)); } diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index df4395395dcde..ec2c3449f6c53 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -50,6 +50,7 @@ #include "sql_alter.h" // Sql_cmd_alter_table* #include "sql_truncate.h" // Sql_cmd_truncate_table #include "sql_admin.h" // Sql_cmd_analyze/Check..._table +#include "sql_backup.h" #include "sql_partition_admin.h" // Sql_cmd_alter_table_*_part. #include "sql_handler.h" // Sql_cmd_handler_* #include "sql_signal.h" @@ -15557,6 +15558,11 @@ backup_statements: /* Table list is empty for unlock */ Lex->sql_command= SQLCOM_BACKUP_LOCK; } + | SERVER_SYM TO_SYM TEXT_STRING_sys + { + Lex->sql_command= SQLCOM_BACKUP_SERVER; + Lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_backup($3); + } ; opt_delete_gtid_domain: diff --git a/sql/sys_vars.inl b/sql/sys_vars.inl index 0f8aa1eb63bf6..8e5f0b984e81e 100644 --- a/sql/sys_vars.inl +++ b/sql/sys_vars.inl @@ -2506,6 +2506,7 @@ public: bool session_update(THD *thd, set_var *var) override; }; +#ifdef HAVE_REPLICATION /* Class for replicate_events_marked_for_skip. We need a custom update function that ensures the slave is stopped when @@ -2647,6 +2648,7 @@ public: } const uchar *global_value_ptr(THD *thd, const LEX_CSTRING *base) const override; }; +#endif /* HAVE_REPLICATION */ /** diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 9e3a23b34ab46..d63751a16af08 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -185,6 +185,8 @@ SET(INNOBASE_SOURCES handler/handler0alter.cc handler/innodb_binlog.cc handler/i_s.cc + handler/backup_innodb.h + handler/backup_innodb.cc ibuf/ibuf0ibuf.cc include/btr0btr.h include/btr0btr.inl diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 326374cd2d194..6ebad0d869aa1 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -35,6 +35,7 @@ Created 11/11/1995 Heikki Tuuri #include "buf0buf.h" #include "buf0checksum.h" #include "buf0dblwr.h" +#include "backup_innodb.h" #include "srv0start.h" #include "page0zip.h" #include "fil0fil.h" @@ -954,6 +955,13 @@ uint32_t fil_space_t::flush_freed(bool writable) noexcept mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex); mysql_mutex_assert_not_owner(&buf_pool.mutex); + /* Note: There is no need to invoke writing_start() or + writing_stop() here, because we are only overwriting freed (garbage) + pages. If backup reads a torn page, it will also have copied a + corresponding FREE_PAGE record, which would be applied on recovery. + Besides, the freed page should never be reachable from other pages + that are part of the snapshot. */ + const bool punch_hole= chain.start->punch_hole == 1; if (!punch_hole && !srv_immediate_scrub_data_uncompressed) return 0; @@ -1229,6 +1237,16 @@ ATTRIBUTE_COLD static size_t buf_flush_LRU_to_withdraw(size_t to_withdraw, return to_withdraw; } +/** Stop writing to a tablespace. +@param space tablespace +@return nullptr */ +static fil_space_t *writing_stop(fil_space_t *space) noexcept +{ + space->writing_stop(); + space->release(); + return nullptr; +} + /** Flush dirty blocks from the end buf_pool.LRU, and move clean blocks to buf_pool.free. @param max maximum number of blocks to flush @@ -1246,6 +1264,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1325,7 +1344,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, buf_pool.lru_hp.set(bpage); mysql_mutex_unlock(&buf_pool.mutex); if (space) - space->release(); + writing_stop(space); auto p= buf_flush_space(space_id); space= p.first; last_space_id= space_id; @@ -1334,6 +1353,10 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, mysql_mutex_lock(&buf_pool.mutex); goto no_space; } + + backup_page_end= space->writing_start() + ? space->backup_page_end() : 0; + mysql_mutex_lock(&buf_pool.mutex); buf_pool.stat.n_pages_written+= p.second; } @@ -1345,8 +1368,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, } else if (space->is_stopping_writes()) { - space->release(); - space= nullptr; + space= writing_stop(space); no_space: mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_discard_page(bpage); @@ -1363,7 +1385,8 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, break; } - if (neighbors && space->is_rotational() && UNIV_LIKELY(!to_withdraw) && + if (neighbors && UNIV_LIKELY(!(to_withdraw | backup_page_end)) && + space->is_rotational() && /* Skip neighbourhood flush from LRU list if we haven't yet reached half of the free page target. */ UT_LIST_GET_LEN(buf_pool.free) * 2 >= free_limit) @@ -1375,10 +1398,15 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, flush: if (UNIV_UNLIKELY(to_withdraw != 0)) to_withdraw= buf_flush_LRU_to_withdraw(to_withdraw, *bpage); - if (bpage->flush(space)) + if (bpage->id().page_no() < backup_page_end) + bpage->lock.u_unlock(true); + else if (bpage->flush(space)) + { ++n->flushed; - else - continue; + goto reacquire_mutex; + } + + continue; } goto reacquire_mutex; @@ -1391,7 +1419,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, buf_pool.lru_hp.set(nullptr); if (space) - space->release(); + writing_stop(space); if (scanned) { @@ -1438,6 +1466,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1509,10 +1538,12 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); mysql_mutex_unlock(&buf_pool.mutex); if (space) - space->release(); + writing_stop(space); auto p= buf_flush_space(space_id); space= p.first; last_space_id= space_id; + backup_page_end= space && space->writing_start() + ? space->backup_page_end() : 0; mysql_mutex_lock(&buf_pool.mutex); buf_pool.stat.n_pages_written+= p.second; mysql_mutex_lock(&buf_pool.flush_list_mutex); @@ -1521,10 +1552,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept ut_ad(!space); } else if (space->is_stopping_writes()) - { - space->release(); - space= nullptr; - } + space= writing_stop(space); if (!space) buf_flush_discard_page(bpage); @@ -1533,9 +1561,15 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); do { - if (neighbors && space->is_rotational()) + if (neighbors && UNIV_LIKELY(!backup_page_end) && + space->is_rotational()) count+= buf_flush_try_neighbors(space, page_id, bpage, neighbors == 1, count, max_n); + else if (bpage->id().page_no() < backup_page_end) + { + bpage->lock.u_unlock(true); + continue; + } else if (bpage->flush(space)) ++count; else @@ -1554,7 +1588,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept buf_pool.flush_hp.set(nullptr); if (space) - space->release(); + writing_stop(space); if (scanned) { @@ -1645,6 +1679,7 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept if (written) buf_pool.stat.n_pages_written+= written; } + mysql_mutex_lock(&buf_pool.flush_list_mutex); for (buf_page_t *bpage= UT_LIST_GET_LAST(buf_pool.flush_list); bpage; ) @@ -1687,17 +1722,29 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept acquired= false; goto was_freed; } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (bpage->flush(space)) + + if (UNIV_UNLIKELY(space->writing_start()) && + bpage->id().page_no() < space->backup_page_end()) + { + bpage->lock.u_unlock(true); + space->writing_stop(); + skip: + mysql_mutex_lock(&buf_pool.mutex); + mysql_mutex_lock(&buf_pool.flush_list_mutex); + may_have_skipped= true; + goto done; + } + + const bool written{bpage->flush(space)}; + space->writing_stop(); + + if (written) { ++n_flush; if (!--max_n_flush) - { - mysql_mutex_lock(&buf_pool.mutex); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - may_have_skipped= true; - goto done; - } + goto skip; mysql_mutex_lock(&buf_pool.mutex); } } @@ -2013,14 +2060,17 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept last_checkpoint_lsn= checkpoint; this->end_lsn= end_lsn; if (!archive) + { + archived_checkpoint= checkpoint; archived_lsn= end_lsn; + } else if (archive_header_was_reset) { ut_ad(resize_log.m_file != log.m_file); /* Make the previous archived log file read-only */ #ifdef _WIN32 resize_log.close(); - SetFileAttributesA(get_archive_path().c_str(), + SetFileAttributesA(get_archive_path(get_first_lsn() - capacity()).c_str(), FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_ARCHIVE); #else struct stat st; @@ -2030,9 +2080,10 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept st.st_mode= 0444; if (fchmod(resize_log.m_file, st.st_mode)) my_error(ER_ERROR_ON_CLOSE, MYF(ME_ERROR_LOG), - get_archive_path().c_str(), errno); + get_archive_path(get_first_lsn() - capacity()).c_str(), errno); resize_log.close(); #endif + innodb_backup_checkpoint(); } else if (resize_log.m_file == log.m_file) { diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc new file mode 100644 index 0000000000000..c97bcb42affdb --- /dev/null +++ b/storage/innobase/handler/backup_innodb.cc @@ -0,0 +1,784 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "sql_class.h" +#include "backup_innodb.h" +#include "sql_backup_interface.h" +#include "trx0trx.h" +#include "buf0flu.h" +#include "log0crypt.h" +#include +#ifdef __linux__ +# include +# include +#endif + +/** Associate a transaction with the current session +@param thd session +@return InnoDB transaction */ +trx_t *check_trx_exists(THD *thd) noexcept; + +namespace +{ +/** Backup state; protected by log_sys.latch */ +class InnoDB_backup +{ + /** pointer to backup context, or nullptr if no backup is active */ + trx_t *trx; + /** the original innodb_log_file_size, or 0 */ + uint64_t old_size; + + /** collection of files to be copied */ + std::vector queue; + /** collection of completed log archive files to be + hard-linked, copied, or moved */ + std::vector logs; + + /** backup target */ + backup_target target; + + /** @return the backup context */ + backup_context &context() const noexcept + { ut_ad(log_sys.latch_have_any()); ut_ad(trx); return trx->lock.backup; } + +public: + /** + Start of BACKUP SERVER: collect all files to be backed up + @param thd current session + @param target backup target + @return error code + @retval 0 on success + */ + int init(THD *thd, backup_target target) noexcept + { + trx_t *trx= check_trx_exists(thd); + if (trx->id || trx->state != TRX_STATE_NOT_STARTED) + { + ut_ad(trx->state != TRX_STATE_BACKUP); + my_error(ER_CANT_DO_THIS_DURING_AN_TRANSACTION, MYF(0)); + return 1; + } + + log_sys.latch.wr_lock(); + ut_ad(!this->trx); + ut_ad(queue.empty()); + if (!logs.empty()) + { + /* A new BACKUP SERVER is being invoked before a previous one + had been fully finalized. Clean up any log files. */ + if (old_size) + delete_logs(); + logs.clear(); + } + + const bool fail{log_sys.backup_start(&old_size, thd)}; + + if (!fail) + { + this->trx= trx; + trx->state= TRX_STATE_BACKUP; + backup_context &ctx{trx->lock.backup}; + ctx.first_lsn= log_sys.get_first_lsn();; + ctx.max_first_lsn= 1; + ctx.first_size= log_sys.file_size; + const lsn_t start= ctx.checkpoint= +#if 1 /* TODO: for incremental backup, allow the start to be specified */ + log_sys.get_latest_checkpoint(ctx.checkpoint_end_lsn); +#else + log_sys.archived_checkpoint; + ctx.checkpoint_end_lsn= log_sys.archived_lsn; +#endif + ctx.last_lsn= 0; + ctx.archived= !old_size; + + this->target= target; + /* Collect all tablespaces that have been created before our + start checkpoint. Newer tablespaces will be recovered by the + innodb_log_archive=ON recovery. + + If a tablespace is deleted before step() is invoked, the file + will not be copied, and a FILE_DELETE record in the log will + ensure correct recovery. + + If a tablespace is renamed between this and end(), the recovery + of a FILE_RENAME record will ensure the correct file name, + no matter which name was used by step(). */ + mysql_mutex_lock(&fil_system.mutex); + for (fil_space_t &space : fil_system.space_list) + if (space.id < SRV_SPACE_ID_UPPER_BOUND && + !space.is_being_imported() && + /* FIXME: how to initialize create_lsn for old files, to + have efficient incremental backup? + fil_node_t::read_page0() cannot assign it from + FIL_PAGE_LSN because that would not reflect the file + creation but for example allocating or freeing a page. + + The easy parts of initializing space->create_lsn are + as follows: + (1) In log_parse_file() when processing FILE_CREATE + (2) In deferred_spaces.create() */ + space.get_create_lsn() < start) + queue.emplace_back(space.id); + mysql_mutex_unlock(&fil_system.mutex); + } + log_sys.latch.wr_unlock(); + DEBUG_SYNC(thd, "innodb_backup_start"); + return fail; + } + + /** + Process a file that was collected at init(). + This may be invoked from multiple concurrent threads. + @param thd current session + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int step(THD *thd) noexcept + { + uint32_t id= FIL_NULL; + lsn_t lsn= 0; + log_sys.latch.wr_lock(); + backup_context &ctx{context()}; + ut_ad(ctx.max_first_lsn); + size_t size{queue.size()}; + if (!logs.empty()) + { + lsn= logs.back(); + if (ctx.max_first_lsn < lsn) + ctx.max_first_lsn= lsn; + logs.pop_back(); + if (!size) + size= logs.size(); + } + else if (size) + { + size--; + id= queue.back(); + queue.pop_back(); + } + log_sys.latch.wr_unlock(); + + if (lsn) + { + if (link_or_move(lsn, nullptr, ctx, target)) + return -1; + } + else if (fil_space_t *space= fil_space_t::get(id)) + { + int res= -1; + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain); node; + node= UT_LIST_GET_NEXT(chain, node)) + if ((res= backup(node))) + break; + space->release(); + if (res) + return res; + } + + size= std::min(size_t{std::numeric_limits::max()}, size); + return int(size); + } + + /** + Finish copying and determine the logical time of the backup snapshot. + fini() must be invoked on the same thd. + @param thd current session + @param abort whether BACKUP SERVER was aborted + @return error code + @retval 0 on success + */ + int end(THD *thd, bool abort) noexcept + { + int fail= 0; + log_sys.latch.wr_lock(); + if (abort) + { + skip_log_dup: + queue.clear(); + if (old_size) + delete_logs(); + logs.clear(); + } + else + { + ut_ad(trx); + ut_ad(queue.empty()); + ut_ad(thd_to_trx(thd) == trx); + if (!trx || trx->state != TRX_STATE_BACKUP) + goto skip_log_dup; + backup_context &ctx= trx->lock.backup; + ut_ad(ctx.max_first_lsn); + ctx.last_lsn= log_sys.get_flushed_lsn(std::memory_order_relaxed); + while (!logs.empty()) + { + const lsn_t lsn{logs.back()}; + if (lsn > ctx.last_lsn) + break; + if (lsn > ctx.max_first_lsn) + ctx.max_first_lsn= lsn; + logs.pop_back(); + log_sys.latch.wr_unlock(); + fail= link_or_move(lsn, nullptr, ctx, target); + log_sys.latch.wr_lock(); + if (fail) + goto skip_log_dup; + } + + { + lsn_t lsn= log_sys.get_first_lsn(); + if (lsn > ctx.max_first_lsn && lsn < ctx.last_lsn) + { + const lsn_t end_lsn{lsn + log_sys.capacity()}; + ctx.max_first_lsn= lsn; + log_sys.latch.wr_unlock(); + if (UNIV_UNLIKELY(ctx.last_lsn > end_lsn)) + { + /* Wait for checkpoint_complete(). */ + buf_flush_sync_batch(end_lsn, true); + lsn= log_get_lsn(); + ut_ad(lsn >= end_lsn); + } + + bool live_hardlink= false; + fail= link_or_move(lsn, &live_hardlink, ctx, target); + log_sys.latch.wr_lock(); + if (fail) + goto skip_log_dup; + if (!live_hardlink) + ctx.max_first_lsn= 0; + } + else + goto skip_log_dup; + } + } + + ut_ad(!log_sys.resize_in_progress()); + ut_ad(log_sys.archive); + + /* FIXME: If we made a hard link to the last log file, + we must retain and restore in fini() a copy of the + checkpoint header in the last log file of our backup. + + Otherwise, the server may overwrite the log header when + that file is still open for writing. The symptoms are as follows + (easy to reproduce by guaranteeing old_size!=0 below + (running the server with innodb_log_archiving=OFF): + + [ERROR] InnoDB: Did not find any checkpoint after LSN=12288 + [Warning] InnoDB: Renaming ib_0000000000003000.log to ib_logfile0 + */ + + if (old_size) + { + log_sys.latch.wr_unlock(); + log_sys.backup_stop_archiving(thd); + log_sys.latch.wr_lock(); + } + + trx= nullptr; + log_sys.backup_stop(old_size, thd); + return fail; + } + + /** + Clean up after end(). + @param thd the parameter that had been passed to end() + @param target backup target + @return error code + @retval 0 on success + */ + int fini(THD *thd, backup_target target) noexcept + { + int fail= 0; + log_sys.latch.wr_lock(); + if (!trx) + { + ut_ad(queue.empty()); + if (old_size) + delete_logs(); + logs.clear(); + } + log_sys.latch.wr_unlock(); + + trx_t *const trx= thd_to_trx(thd); + if (!trx || trx->state != TRX_STATE_BACKUP) + ut_ad("invalid state" == 0); + else + { + ut_ad(!trx->id); + const backup_context &ctx{trx->lock.backup}; + if (ctx.max_first_lsn) + { + const lsn_t size= + std::max(log_sys.FILE_SIZE_MIN, log_sys.START_OFFSET + + (((ctx.last_lsn - ctx.max_first_lsn) + 4095) & + ~4095ULL)); + /* Copy our clone of the last log until the final LSN */ +#ifdef _WIN32 + std::string src{target.path}; + src.push_back('/'); + std::string dst{src}; + src.append("ib_logfile101"); + log_sys.append_archive_name(dst, ctx.max_first_lsn); + const char *s= src.c_str(), *d= dst.c_str(); + if (!CopyFileEx(s, d, nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING) || + !SetFileAttributes(s, FILE_ATTRIBUTE_NORMAL) || + !DeleteFile(s)) + { + fail: + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), s, d, errno); + fail= 1; + } + /* Trim the file to the desired size after copying it. */ + HANDLE dh= CreateFile(d, GENERIC_READ, FILE_SHARE_READ, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (dh == INVALID_HANDLE_VALUE) + goto fail; + + LARGE_INTEGER li; + li.QuadPart= size; + BOOL ok= SetFilePointerEx(dh, li, nullptr, FILE_BEGIN) && + (ctx.max_first_lsn != ctx.first_lsn || + !write_checkpoint(dh, ctx.checkpoint_end_lsn - ctx.first_lsn + + log_sys.START_OFFSET)); + CloseHandle(dh); + if (!ok) + goto fail; +#else + ut_ad(target.directory); + int s= openat(target.fd, "ib_logfile101", O_RDONLY); + std::string dst; + log_sys.append_archive_name(dst, ctx.max_first_lsn); + int d{-1}; + if (s == -1) + { + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), "ib_logfile101", + errno); + fail= 1; + goto done; + } + ut_ad(target.directory); + d= openat(target.fd, dst.c_str(), + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (d < 0) + { + fail: + fail= 1; + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), + "ib_logfile101", dst.c_str(), errno); + close(s); + } + else + { + fail= copy_file(s, d, log_sys.START_OFFSET, size) || + (ctx.max_first_lsn == ctx.first_lsn && + write_checkpoint(d, ctx.checkpoint_end_lsn - ctx.first_lsn + + log_sys.START_OFFSET)); + if (close(d) || fail) + goto fail; + if (unlinkat(target.fd, "ib_logfile101", 0)) + { + my_error(ER_CANT_DELETE_FILE, MYF(ME_ERROR_LOG), + "ib_logfile101", errno); + fail= 1; + } + std::ignore= close(s); + } + done:; +#endif + /* TODO: punch hole to the start of the first log file + if we had old_size!=0 */ + sql_print_information("innodb_log_recovery_start=" LSN_PF + " (checkpoint " LSN_PF ")", + trx->lock.backup.checkpoint_end_lsn, + trx->lock.backup.checkpoint); + } + trx->lock.backup= {}; + trx->state= TRX_STATE_NOT_STARTED; + } + return fail; + } + + /** + Complete the first checkpoint in a new archive log file. + */ + void checkpoint_complete() noexcept + { + ut_ad(log_sys.latch_have_wr()); + if (trx) + logs.emplace_back(log_sys.get_first_lsn() - log_sys.capacity()); + } + +private: + /** Safely start backing up a tablespace file */ + static void backup_start(fil_space_t *space) noexcept + { + if (space->backup_start(space->size)) + os_aio_wait_until_no_pending_writes(false); + } + /* Stop backing up a tablespace */ + static void backup_stop(fil_space_t *space) noexcept + { space->backup_stop(); } + + /** Delete unnecessary logs that had been created for backup. */ + void delete_logs() noexcept + { + ut_ad(old_size); + for (const lsn_t lsn : logs) + IF_WIN(DeleteFile,unlink)(log_sys.get_archive_path(lsn).c_str()); + } + + /** + Back up a persistent InnoDB data file. + @param node InnoDB data file + */ + int backup(fil_node_t *node) noexcept + { + for (bool tried_mkdir{false};;) + { +#ifdef _WIN32 + std::string path{target.path}; + path.push_back('/'); + backup_start(node->space); + path.append(node->name); + bool ok= CopyFileEx(node->name, path.c_str(), nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING); + backup_stop(node->space); + if (!ok) + { + unsigned long err= GetLastError(); + if (err == ERROR_PATH_NOT_FOUND && !tried_mkdir && + node->space->id && !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + path.erase(path.rfind('/')); + if (CreateDirectory(path.c_str(), + my_dir_security_attributes.lpSecurityDescriptor + ? &my_dir_security_attributes : nullptr) || + (err= GetLastError()) == ERROR_ALREADY_EXISTS) + continue; + } + + my_osmaperr(err); + goto fail; + } + break; +#else + int f; + ut_ad(target.directory); +# ifdef __APPLE__ + backup_start(node->space); + f= fclonefileat(node->handle, target.fd, node->name, 0); + backup_stop(node->space); + if (!f) + break; + switch (errno) { + case ENOENT: + goto try_mkdir; + case ENOTSUP: + break; + default: + goto fail; + } +# endif + f= openat(target.fd, node->name, + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (f < 0) + { + if (errno == ENOENT) + { +# ifdef __APPLE__ + try_mkdir: +# endif + if (!tried_mkdir && node->space->id && + !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + const char *sep= strchr(node->name, '/'); + ut_ad(sep); + sep= strchr(sep + 1, '/'); + ut_ad(sep); + std::string dir{node->name, size_t(sep - node->name)}; + if (!mkdirat(target.fd, dir.c_str(), 0777) || errno == EEXIST) + continue; + } + } + goto fail; + } + + /* TODO: page range locking; avoid copying freed page ranges */ + backup_start(node->space); + int err= copy_file(node->handle, f, 0, + off_t{node->size} * node->space->physical_size()); + backup_stop(node->space); + if (close(f) || err) + goto fail; + break; +#endif + } + return 0; + fail: + my_error(ER_CANT_CREATE_FILE, MYF(0), node->name, errno); + return -1; + } + + /** Write a checkpoint header pointing to the start of the backup. + @param dst target file + @param c offset of the FILE_CHECKPOINT mini-transaction + @return error code + @retval 0 on success */ + static int write_checkpoint(int dst, uint64_t c) noexcept + { +#ifdef _WIN32 + using tpool::pwrite; +#endif + uint64_t buf[8]{}; + ut_ad(c >= log_sys.START_OFFSET); + if (log_sys.is_encrypted()) + log_crypt_write_header(reinterpret_cast(buf), true); + buf[4 * log_sys.is_encrypted()]= my_htobe64(c); + + for (ssize_t o= 0, count= sizeof buf; count;) + { + ssize_t ret= + pwrite(dst, reinterpret_cast(buf) + o, count, o); + if (ret <= 0 || ret > count) + return -1; + o+= ret; + count-= ret; + } + return 0; + } + +#ifndef _WIN32 + /** Copy a log file. + @param src source file + @param dst target file + @param start start offset of record payload in the log + @param size end offset of the log + @param ctx backup context + @param c offset of the FILE_CHECKPOINT mini-transaction + @return error code + @retval 0 on success */ + static int copy_log_file_part(int src, int dst, off_t start, off_t size, + const backup_context &ctx, uint64_t c) + noexcept + { + if (int err= copy_file(src, dst, start, size)) + return err; + return write_checkpoint(dst, c); + } +#endif + + /** Hard-link (copy) or rename (move) an archive log file. + @param lsn The first LSN in the file + @param clone pointer to a flag that will be set if a live log was + hard-linked (needing deduplication), + or nullptr if the source log file is known to be read-only + @param ctx backup context + @param target backup target + @return error code + @retval 0 on success */ + static int link_or_move(lsn_t lsn, bool *clone, + const backup_context &ctx, + IF_WIN(const backup_target&,backup_target) target) + noexcept + { + ut_ad(!clone || !*clone); + const std::string p{log_sys.get_archive_path(lsn)}; + const char *const path= p.c_str(), *basename= strrchr(path, '/'); + if (!basename) + basename= path; + else + basename++; + const bool move{!clone && !ctx.archived}; + +#ifdef _WIN32 + std::string b{target.path}; + b.push_back('/'); + b.append(clone ? "ib_logfile101" : basename); + const char *destname= b.c_str(); + + unsigned long err; + if (move) + { + if (!MoveFileEx(path, destname, MOVEFILE_COPY_ALLOWED)) + { + fail: + err= GetLastError(); + got_err: + my_osmaperr(err); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), path, basename, errno); + return -1; + } +# if 0 + if (lsn < ctx.checkpoint) + /* FIXME: punch hole */; +# endif + } + else if (!CreateHardLink(destname, path, nullptr)) + { + if ((err= GetLastError()) != ERROR_NOT_SAME_DEVICE) + goto got_err; + /* Hard-linking failed. Try copying with the final name. */ + b= target.path; + b.push_back('/'); + b.append(basename); + destname= b.c_str(); + + if (!CopyFileEx(path, destname, nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING)) + goto fail; + } + else if (clone) + *clone= true; +#else + ut_ad(target.directory); + if (move + ? !renameat(AT_FDCWD, path, target.fd, basename) + : !linkat(AT_FDCWD, path, target.fd, + clone ? "ib_logfile101" : basename, AT_SYMLINK_FOLLOW)) + { +#ifdef __linux__ + if (!move || lsn != ctx.first_lsn); + else if (off_t garbage= (ctx.checkpoint - lsn) & ~4095ULL) + /* Best effort to punch a hole to free up some garbage in + the first file. We do not care about failures. */ + if (!fchmodat(target.fd, basename, 0644, 0)) + { + int dst= openat(target.fd, basename, O_RDWR); + if (dst >= 0) + fallocate(dst, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + log_sys.START_OFFSET, garbage); + close(dst); + fchmodat(target.fd, basename, 0444, 0); + } +#endif + if (clone) + *clone= !move; + return 0; + } + else if (errno != EXDEV) + { + fail: + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), path, basename, errno); + return -1; + } + else + { + int src= open(path, O_RDONLY); + if (src < 0) + goto fail; + if (move && unlink(path)) + { + close_and_fail: + std::ignore= close(src); + goto fail; + } + int dst= openat(target.fd, basename, + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (dst < 0) + goto close_and_fail; + int err= lsn >= ctx.checkpoint + ? copy_entire_file(src, dst) + : copy_log_file_part(src, dst, + log_sys.START_OFFSET + + (((ctx.checkpoint - lsn) + 4095) & ~4095ULL), + ctx.first_size, ctx, + ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + + if ((close(dst) | close(src)) || err) + goto fail; + } +#endif + return 0; + } +}; + +/** The backup context; protected by log_sys.latch */ +static InnoDB_backup innodb_backup; +} + +bool log_t::backup_start(uint64_t *old_size, THD *thd) noexcept +{ + ut_ad(latch_have_wr()); + ut_ad(!backup); + backup= true; + *old_size= 0; + if (archive) + return false; + const uint64_t old_file_size{file_size}; + latch.wr_unlock(); + const bool fail{set_archive(true, thd, true)}; + latch.wr_lock(); + if (!fail) + { + *old_size= old_file_size; + return false; + } + ut_ad(backup); + backup= false; + const uint64_t new_file_size{file_size}; + latch.wr_unlock(); + if (old_file_size != new_file_size && old_file_size && + resize_start(old_file_size, thd) == RESIZE_STARTED) + resize_finish(thd); + latch.wr_lock(); + return true; +} + +void log_t::backup_stop(uint64_t old_size, THD *thd) noexcept +{ + ut_ad(latch_have_wr()); + /* We will be invoked with old_size=0 after a failed backup_start(), + or if innodb_log_archive=ON held during a successful backup_start(). */ + ut_ad(!old_size || !resize_in_progress()); + ut_ad(!old_size || backup); + backup= false; + const uint64_t new_size{file_size}; + latch.wr_unlock(); + if (old_size && old_size != new_size && + resize_start(old_size, thd) == RESIZE_STARTED) + resize_finish(thd); +} + +int innodb_backup_start(THD *thd, backup_target target) noexcept +{ + return innodb_backup.init(thd, target); +} + +int innodb_backup_step(THD *thd) noexcept +{ + return innodb_backup.step(thd); +} + +int innodb_backup_end(THD *thd, bool abort) noexcept +{ + return innodb_backup.end(thd, abort); +} + +int innodb_backup_finalize(THD *thd, backup_target target) noexcept +{ + return innodb_backup.fini(thd, target); +} + +void innodb_backup_checkpoint() noexcept +{ + innodb_backup.checkpoint_complete(); +} diff --git a/storage/innobase/handler/backup_innodb.h b/storage/innobase/handler/backup_innodb.h new file mode 100644 index 0000000000000..8d5f81ca35898 --- /dev/null +++ b/storage/innobase/handler/backup_innodb.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/** + Start of BACKUP SERVER: collect all files to be backed up + @param thd current session + @param target backup target + @return error code + @retval 0 on success +*/ +int innodb_backup_start(THD *thd, backup_target target) noexcept; + +/** + Process a file that was collected in backup_start(). + @param thd current session + @return number of files remaining, or negative on error + @retval 0 on completion +*/ +int innodb_backup_step(THD *thd) noexcept; + +/** + Finish copying and determine the logical time of the backup snapshot. + @param thd current session + @param abort whether BACKUP SERVER was aborted + @return error code + @retval 0 on success +*/ +int innodb_backup_end(THD *thd, bool abort) noexcept; + +/** + Clean up after innodb_backup_end(). + @param thd the parameter on which innodb_backup_end() had been invoked + @param target backup target + @return error code + @retval 0 on success +*/ +int innodb_backup_finalize(THD *thd, backup_target target) noexcept; + +/** + Complete the first checkpoint in a new archive log file. +*/ +void innodb_backup_checkpoint() noexcept; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 3a05963b4081e..5c28d270d13ff 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -152,6 +152,7 @@ MDL_ticket *get_mdl_ticket(TABLE *table) noexcept; #include "ha_innodb.h" #include "i_s.h" +#include "backup_innodb.h" #include #include @@ -2606,16 +2607,10 @@ innobase_trx_allocate( DBUG_RETURN(trx); } -/*********************************************************************//** -Gets the InnoDB transaction handle for a MySQL handler object, creates -an InnoDB transaction struct if the corresponding MySQL thread struct still -lacks one. -@return InnoDB transaction handle */ -static -trx_t* -check_trx_exists( -/*=============*/ - THD* thd) /*!< in: user thread handle */ +/** Associate a transaction with the current session +@param thd session +@return InnoDB transaction */ +trx_t *check_trx_exists(THD *thd) noexcept { if (trx_t* trx = thd_to_trx(thd)) { ut_a(trx->magic_n == TRX_MAGIC_N); @@ -4148,6 +4143,10 @@ static int innodb_init(void* p) = innodb_prepare_commit_versioned; innobase_hton->update_optimizer_costs= innobase_update_optimizer_costs; + innobase_hton->backup_start = innodb_backup_start; + innobase_hton->backup_step = innodb_backup_step; + innobase_hton->backup_end = innodb_backup_end; + innobase_hton->backup_finalize = innodb_backup_finalize; innobase_hton->binlog_init= innodb_binlog_init; innobase_hton->set_binlog_max_size= ibb_set_max_size; innobase_hton->binlog_write_direct_ordered= @@ -18816,39 +18815,7 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, ib_senderrf(thd, IB_LOG_LEVEL_ERROR, ER_CANT_CREATE_HANDLER_FILE); break; case log_t::RESIZE_STARTED: - for (timespec abstime;;) - { - if (thd_kill_level(thd)) - { - log_sys.resize_abort(thd); - break; - } - - set_timespec(abstime, 5); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - lsn_t resizing= log_sys.resize_in_progress(); - if (resizing > buf_pool.get_oldest_modification(0)) - { - buf_pool.page_cleaner_wakeup(true); - my_cond_timedwait(&buf_pool.done_flush_list, - &buf_pool.flush_list_mutex.m_mutex, &abstime); - resizing= log_sys.resize_in_progress(); - } - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (!resizing || !log_sys.resize_running(thd)) - break; - log_sys.latch.wr_lock(); - while (resizing > log_sys.get_lsn()) - { - ut_ad(!log_sys.is_mmap()); - /* The server is almost idle. Write dummy FILE_CHECKPOINT records - to ensure that the log resizing will complete. */ - mtr_t mtr{nullptr}; - mtr.start(); - mtr.commit_files(log_sys.last_checkpoint_lsn); - } - log_sys.latch.wr_unlock(); - } + log_sys.resize_finish(thd); } } mysql_mutex_lock(&LOCK_global_system_variables); @@ -19703,7 +19670,9 @@ static MYSQL_SYSVAR_BOOL(data_file_write_through, fil_system.write_through, static void innodb_log_archive_update(THD *thd, st_mysql_sys_var*, void *, const void *save) noexcept { + mysql_mutex_unlock(&LOCK_global_system_variables); log_sys.set_archive(*static_cast(save), thd); + mysql_mutex_lock(&LOCK_global_system_variables); } static MYSQL_SYSVAR_BOOL(log_archive, log_sys.archive, @@ -19716,10 +19685,20 @@ static MYSQL_SYSVAR_UINT64_T(log_archive_start, innodb_log_archive_start, "initial value of innodb_lsn_archived; 0=auto-detect", nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); +static void innodb_log_recovery_start_update(THD *, st_mysql_sys_var*, + void *, const void *save) noexcept +{ + const lsn_t lsn{*static_cast(save)}; + recv_sys.recovery_start= lsn; + if (lsn && log_sys.archive) + log_sys.archived_checkpoint= lsn; +} + static MYSQL_SYSVAR_UINT64_T(log_recovery_start, recv_sys.recovery_start, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + PLUGIN_VAR_RQCMDARG, "LSN to start recovery from (0=automatic)", - nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); + nullptr, innodb_log_recovery_start_update, + 0, 0, std::numeric_limits::max(), 0); static MYSQL_SYSVAR_UINT64_T(log_recovery_target, recv_sys.rpo, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index a43020daf7b63..06fb28dd98331 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -408,6 +408,13 @@ struct fil_space_t final /** Whether any corruption of this tablespace has been reported */ mutable std::atomic_flag is_corrupted= ATOMIC_FLAG_INIT; + /** BACKUP SERVER flag in write_or_backup */ + static constexpr uint8_t BACKUP{128}; + /** whether there is a pending write or backup */ + std::atomic write_or_backup{0}; + /** first page number that is not being backed up */ + std::atomic backup_end{0}; + public: /** mutex to protect freed_ranges and last_freed_lsn */ std::mutex freed_range_mutex; @@ -1044,6 +1051,43 @@ struct fil_space_t final VALIDATE_IMPORT }; + /** Note that writes are being submitted to the tablespace. + @return whether a backup is pending */ + bool writing_start() noexcept + { + uint8_t wb{write_or_backup.fetch_add(1, std::memory_order_acq_rel)}; + ut_ad(~wb & (BACKUP - 1)); + return wb & BACKUP; + } + + /** Note that we there are no more pending writes to the tablespace. */ + void writing_stop() noexcept + { + ut_d(uint8_t wb=) write_or_backup.fetch_sub(1, std::memory_order_release); + ut_ad(wb & ~BACKUP); + } + + /** Note that we backing up some pages of the underlying files. + @param last_page the last page that is being backed up */ + bool backup_start(uint32_t last_page) noexcept + { + backup_end.store(last_page, std::memory_order_relaxed); + uint8_t wb{write_or_backup.fetch_add(BACKUP, std::memory_order_acq_rel)}; + ut_ad(!(wb & BACKUP)); + return wb & ~BACKUP; + } + /** Note that we are not currently backing up the underlying files. */ + void backup_stop() noexcept + { + backup_end.store(0, std::memory_order_relaxed); + ut_d(uint8_t wb=) + write_or_backup.fetch_sub(BACKUP, std::memory_order_release); + ut_ad(wb & BACKUP); + } + /** @return the first page number that is not being backed up */ + uint32_t backup_page_end() const noexcept + { return backup_end.load(std::memory_order_relaxed); } + /** Update the data structures on write completion */ void complete_write() noexcept; diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 44a827dbf636d..6d07fa25e013d 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -221,6 +221,8 @@ struct log_t /** whether !archive log records may have been written with get_sequence_bit()==0 */ bool circular_recovery_from_sequence_bit_0:1; + /** whether we are between backup_start() and backup_stop() */ + bool backup:1; public: /** the default value of log_mmap */ static constexpr bool log_mmap_default= @@ -288,6 +290,8 @@ struct log_t Atomic_relaxed last_checkpoint_lsn; /** The log writer (protected by latch.wr_lock()) */ lsn_t (*writer)() noexcept; + /** the earliest available checkpoint; protected by latch.wr_lock() */ + lsn_t archived_checkpoint; /** end_lsn of the first available checkpoint, or 0; protected by latch.wr_lock() */ lsn_t archived_lsn; @@ -369,11 +373,24 @@ struct log_t RESIZE_NO_CHANGE, RESIZE_IN_PROGRESS, RESIZE_STARTED, RESIZE_FAILED }; +private: /** Start resizing the log and release the exclusive latch. + @param size requested new file_size + @param thd the current thread identifier + @param backup whether the caller is backup_start() or backup_stop() + @return whether the resizing was started successfully */ + resize_start_status resize_start(uint64_t size, void *thd, bool backup) + noexcept; +public: + /** Start resizing the log. @param size requested new file_size @param thd the current thread identifier @return whether the resizing was started successfully */ - resize_start_status resize_start(os_offset_t size, void *thd) noexcept; + resize_start_status resize_start(uint64_t size, void *thd) noexcept + { return resize_start(size, thd, false); } + + /** Wait for the completion of resize_start() == RESIZE_STARTED */ + void resize_finish(THD *thd) noexcept; /** Abort a resize_start() that we started. @param thd thread identifier that had been passed to resize_start() */ @@ -397,10 +414,37 @@ struct log_t resize_write_low(lsn, end, len, seq); } +private: + /** SET GLOBAL innodb_log_archive, or start/stop BACKUP SERVER + @param archive the new value of innodb_log_archive + @param thd SQL connection + @param backup whether the caller is backup_start() or backup_stop() + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd, bool backup) noexcept; +public: /** SET GLOBAL innodb_log_archive @param archive the new value of innodb_log_archive - @param thd SQL connection */ - void set_archive(my_bool archive, THD *thd) noexcept; + @param thd SQL connection + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd) noexcept + { return set_archive(archive, thd, false); } + + /** Start BACKUP SERVER. + @param old_size the old file_size, or 0 on failure or when + already running innodb_log_archive=ON + @param thd SQL connection + @return whether the operation failed */ + bool backup_start(uint64_t *old_size, THD *thd) noexcept; + /** Stop log archiving in BACKUP SERVER clean-up + @param thd SQL connection + @return whether the operation failed */ + bool backup_stop_archiving(THD *thd) noexcept + { return set_archive(false, thd, true); } + + /** Stop BACKUP SERVER. + @param old_size the value returned by backup_start() + @param thd SQL connection */ + void backup_stop(uint64_t old_size, THD *thd) noexcept; private: /** Replicate a write to the log. @@ -695,6 +739,18 @@ struct log_t /** @return the first LSN of the log file */ lsn_t get_first_lsn() const noexcept { return first_lsn; } + /** + Determine the latest checkpoint. + @param end LSN leading to the FILE_CHECKPOINT record + @return the latest checkpoint LSN + */ + lsn_t get_latest_checkpoint(lsn_t &end) const noexcept + { + ut_ad(latch_have_any()); + end= end_lsn; + return last_checkpoint_lsn; + } + /** Set the recovered checkpoint. @param lsn log sequence number of the checkpoint @param end_lsn LSN passed to write_checkpoint() diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 9a9bd152bd0ed..d09a1ffe17f9e 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -348,10 +348,14 @@ struct trx_lock_t only be modified by the thread that is serving the running transaction. */ - /** Pre-allocated record locks */ - struct { - alignas(CPU_LEVEL1_DCACHE_LINESIZE) ib_lock_t lock; - } rec_pool[8]; + union + { + /** Context for finalizing BACKUP SERVER */ + backup_context backup; + + /** Pre-allocated record locks */ + struct { alignas(CPU_LEVEL1_DCACHE_LINESIZE) ib_lock_t lock; } rec_pool[8]; + }; /** Pre-allocated table locks */ ib_lock_t table_pool[8]; diff --git a/storage/innobase/include/trx0trx.inl b/storage/innobase/include/trx0trx.inl index 317f1f5cd0d27..4aff8ef96c58f 100644 --- a/storage/innobase/include/trx0trx.inl +++ b/storage/innobase/include/trx0trx.inl @@ -68,6 +68,7 @@ trx_state_eq( return(true); case TRX_STATE_ABORTED: + case TRX_STATE_BACKUP: break; } ut_error; diff --git a/storage/innobase/include/trx0types.h b/storage/innobase/include/trx0types.h index 6fde1e831e5bd..300f8c8e9f4ab 100644 --- a/storage/innobase/include/trx0types.h +++ b/storage/innobase/include/trx0types.h @@ -62,7 +62,28 @@ enum trx_state_t { /** XA PREPARE transaction that was returned to ha_recover() */ TRX_STATE_PREPARED_RECOVERED, /** The transaction has been committed (or completely rolled back) */ - TRX_STATE_COMMITTED_IN_MEMORY + TRX_STATE_COMMITTED_IN_MEMORY, + /** The transaction holds context for BACKUP SERVER */ + TRX_STATE_BACKUP +}; + +/** TRX_STATE_BACKUP context */ +struct backup_context +{ + /** Start LSN of the first backed up log file */ + lsn_t first_lsn; + /** Start LSN of the latest copied log file, or 1 if none yet */ + lsn_t max_first_lsn; + /** size of the first log file */ + uint64_t first_size; + /** Checkpoint at the start of the backup */ + lsn_t checkpoint; + /** Log record pointing to the checkpoint */ + lsn_t checkpoint_end_lsn; + /** Final LSN of the backup */ + lsn_t last_lsn; + /** the original state of innodb_log_archive before/after backup */ + bool archived; }; /** Transaction bulk insert operation @see trx_t::bulk_insert */ diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index aad4f768f0867..f4519aa6767a7 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -634,7 +634,7 @@ void log_t::set_buffered(bool buffered) noexcept } #endif - /** Try to enable or disable durable writes (update log_write_through) */ +/** Try to enable or disable durable writes (update log_write_through) */ void log_t::set_write_through(bool write_through) { if (is_mmap() || high_level_read_only || recv_sys.rpo) @@ -763,9 +763,12 @@ void log_t::header_rewrite(my_bool archive) noexcept /** SET GLOBAL innodb_log_archive @param archive the new value of innodb_log_archive -@param thd SQL connection */ -void log_t::set_archive(my_bool archive, THD *thd) noexcept +@param thd SQL connection +@param backup whether the caller is backup_start() or backup_stop() +@return whether the operation failed */ +bool log_t::set_archive(my_bool archive, THD *thd, bool backup) noexcept { + bool fail= false; thd_wait_begin(thd, THD_WAIT_DISKIO); tpool::tpool_wait_begin(); lsn_t wait_lsn; @@ -779,12 +782,20 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept my_printf_error(ER_WRONG_USAGE, "SET GLOBAL innodb_log_file_size is in progress", MYF(0)); + fail: + fail= true; + wait_lsn= 0; break; } if (archive == this->archive) break; - if (thd_kill_level(thd)) - break; + if ((!backup || archive) && thd_kill_level(thd)) + goto fail; + if (!backup && this->backup) + { + my_printf_error(ER_WRONG_USAGE, "BACKUP SERVER is in progress", MYF(0)); + goto fail; + } if (resize_log.is_opened()) { @@ -893,7 +904,7 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept if (!log.is_opened()) { my_error(ER_ERROR_ON_READ, MYF(0), old_name, errno); - break; + goto fail; } } #endif @@ -918,7 +929,7 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept { my_error(ER_ERROR_ON_RENAME, MYF(0), old_name, new_name, my_errno); first_lsn= old_first_lsn; - break; + goto fail; } if (archive) @@ -950,14 +961,16 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept thd_wait_end(thd); if (wait_lsn) mtr_flush_ahead(wait_lsn); + return fail; } -/** Start resizing the log and release the exclusive latch. -@param size requested new file_size -@param thd the current thread identifier +/** Start resizing the log. +@param size requested new file_size +@param thd the current thread identifier +@param backup whether the caller is backup_start() or backup_stop() @return whether the resizing was started successfully */ -log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) - noexcept +log_t::resize_start_status log_t::resize_start(uint64_t size, void *thd, + bool backup) noexcept { ut_ad(size >= 4U << 20); ut_ad(!(size & 4095)); @@ -988,6 +1001,9 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) resize_target= size; } } + else if (!backup && this->backup) + /* backup_start() or backup_stop() is running */ + status= RESIZE_FAILED; else { lsn_t start_lsn; @@ -1089,6 +1105,44 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) return status; } +/** Wait for the completion of resize_start() == RESIZE_STARTED */ +void log_t::resize_finish(THD *thd) noexcept +{ + for (timespec abstime;;) + { + if (thd_kill_level(thd)) + { + resize_abort(thd); + break; + } + + set_timespec(abstime, 5); + mysql_mutex_lock(&buf_pool.flush_list_mutex); + lsn_t resizing= resize_in_progress(); + if (resizing > buf_pool.get_oldest_modification(0)) + { + buf_pool.page_cleaner_wakeup(true); + my_cond_timedwait(&buf_pool.done_flush_list, + &buf_pool.flush_list_mutex.m_mutex, &abstime); + resizing= resize_in_progress(); + } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + if (!resizing || !resize_running(thd)) + break; + latch.wr_lock(); + while (resizing > get_lsn()) + { + ut_ad(!is_mmap()); + /* The server is almost idle. Write dummy FILE_CHECKPOINT records + to ensure that the log resizing will complete. */ + mtr_t mtr{nullptr}; + mtr.start(); + mtr.commit_files(last_checkpoint_lsn); + } + latch.wr_unlock(); + } +} + /** Abort a resize_start() that we started. */ void log_t::resize_abort(void *thd) noexcept { diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 12b0715e439bc..151920411bf8b 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2099,6 +2099,7 @@ dberr_t recv_sys_t::find_checkpoint() memset_aligned<4096>(const_cast(field_ref_zero), 0, 4096); /* Mark the redo log for upgrading. */ lsn= file_checkpoint= log_sys.last_checkpoint_lsn; + log_sys.archived_checkpoint= lsn; log_sys.set_recovered_lsn(lsn); if (rpo && rpo != lsn) { @@ -2177,7 +2178,8 @@ dberr_t recv_sys_t::find_checkpoint() log_sys.set_recovered_checkpoint(checkpoint_lsn, lsn= end_lsn, field == log_t::CHECKPOINT_1); } - if (!log_sys.last_checkpoint_lsn) + log_sys.archived_checkpoint= log_sys.last_checkpoint_lsn; + if (!log_sys.archived_checkpoint) goto got_no_checkpoint; else if (!log_sys.archived_lsn) log_sys.archived_lsn= lsn; diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 6494c5e21b96e..4ef86095f78a3 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2017,7 +2017,15 @@ os_file_create_func( ); DWORD create_flag = OPEN_EXISTING; - DWORD share_mode = read_only + /* BACKUP SERVER may invoke CreateHardLink() on a log file that + may concurrently be written to. This is why we must allow + FILE_SHARE_WRITE. This has the side effect that multiple InnoDB + instances may be concurrently started on the same log file. + However, InnoDB will not write any log before it has successfully + opened data files. As long as the multiple instances are also + opening the same InnoDB data files (such as the system tablespace), + they should fail to start up concurrently. */ + DWORD share_mode = read_only || type == OS_LOG_FILE ? FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE : FILE_SHARE_READ | FILE_SHARE_DELETE; diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index 11d411d7a42f2..02e78965c1383 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -174,6 +174,7 @@ dberr_t trx_t::rollback(const undo_no_t *savept) noexcept case TRX_STATE_PREPARED: case TRX_STATE_PREPARED_RECOVERED: case TRX_STATE_COMMITTED_IN_MEMORY: + case TRX_STATE_BACKUP: ut_ad("invalid state" == 0); /* fall through */ case TRX_STATE_ACTIVE: @@ -260,6 +261,8 @@ dberr_t trx_rollback_for_mysql(trx_t* trx) case TRX_STATE_COMMITTED_IN_MEMORY: ut_ad(!trx->is_autocommit_non_locking()); break; + case TRX_STATE_BACKUP: + break; } ut_error; diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index 2f2265a3df1fd..7c67841911bac 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -54,6 +54,7 @@ void rw_trx_hash_t::validate_element(trx_t *trx) switch (trx->state) { case TRX_STATE_NOT_STARTED: case TRX_STATE_ABORTED: + case TRX_STATE_BACKUP: ut_error; case TRX_STATE_PREPARED: case TRX_STATE_PREPARED_RECOVERED: @@ -380,6 +381,7 @@ size_t trx_sys_t::any_active_transactions(size_t *prepared) switch (trx.state) { case TRX_STATE_NOT_STARTED: case TRX_STATE_ABORTED: + case TRX_STATE_BACKUP: break; case TRX_STATE_ACTIVE: if (!trx.id) diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 551819aef4cf0..9f45dfbe8930d 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1660,6 +1660,7 @@ trx_commit_or_rollback_prepare( case TRX_STATE_COMMITTED_IN_MEMORY: case TRX_STATE_ABORTED: + case TRX_STATE_BACKUP: break; } @@ -1744,6 +1745,7 @@ void trx_commit_for_mysql(trx_t *trx) noexcept trx->op_info= ""; break; case TRX_STATE_COMMITTED_IN_MEMORY: + case TRX_STATE_BACKUP: ut_error; break; } @@ -1813,6 +1815,9 @@ trx_print_low( case TRX_STATE_COMMITTED_IN_MEMORY: fputs(", COMMITTED IN MEMORY", f); goto state_ok; + case TRX_STATE_BACKUP: + fputs(", BACKUP SERVER", f); + goto state_ok; } fprintf(f, ", state %lu", (ulong) trx->state); ut_ad(0); @@ -2172,6 +2177,7 @@ trx_start_if_not_started_xa_low( case TRX_STATE_PREPARED: case TRX_STATE_PREPARED_RECOVERED: case TRX_STATE_COMMITTED_IN_MEMORY: + case TRX_STATE_BACKUP: break; } @@ -2201,6 +2207,7 @@ trx_start_if_not_started_low( case TRX_STATE_PREPARED: case TRX_STATE_PREPARED_RECOVERED: case TRX_STATE_COMMITTED_IN_MEMORY: + case TRX_STATE_BACKUP: break; } From c0e48fc966d3d3926bade68c7c5bfdb9fe88ad3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 29 May 2026 17:14:01 +0300 Subject: [PATCH 02/25] WIP MDEV-39092, and back up non-InnoDB files --- mysql-test/main/backup_server_restore.result | 41 +++ mysql-test/main/backup_server_restore.test | 38 ++ mysql-test/suite/backup/backup_innodb.test | 14 +- storage/maria/CMakeLists.txt | 1 + storage/maria/ha_maria.cc | 4 + storage/maria/ma_backup.cc | 362 +++++++++++++++++++ storage/maria/ma_backup.h | 47 +++ 7 files changed, 502 insertions(+), 5 deletions(-) create mode 100644 mysql-test/main/backup_server_restore.result create mode 100644 mysql-test/main/backup_server_restore.test create mode 100644 storage/maria/ma_backup.cc create mode 100644 storage/maria/ma_backup.h diff --git a/mysql-test/main/backup_server_restore.result b/mysql-test/main/backup_server_restore.result new file mode 100644 index 0000000000000..83bf230bc8619 --- /dev/null +++ b/mysql-test/main/backup_server_restore.result @@ -0,0 +1,41 @@ +Prepare database +CREATE TABLE tinno (i INTEGER) ENGINE=InnoDB; +INSERT INTO tinno VALUES (1), (2), (3), (4); +CREATE TABLE tariatr (i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO tariatr VALUES (2), (3), (5), (7); +CREATE TABLE tariant (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO tariant VALUES (1), (1), (2), (3), (5); +Back up the database +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +Restore the database +# restart: --datadir=MYSQLTEST_VARDIR/some_directory +Check contents after restore +SELECT * FROM tinno; +i +1 +2 +3 +4 +SELECT * FROM tariatr; +i +2 +3 +5 +7 +SELECT * FROM tariant; +i +1 +1 +2 +3 +5 +Warnings: +Error 145 Got error '145 "Table was marked as crashed and should be repaired"' for './test/tariant' +Warning 1034 1 client is using or hasn't closed the table properly +Note 1034 Table is fixed +Restart database in original data directory +# restart +Clean up +DROP TABLE tinno; +DROP TABLE tariatr; +DROP TABLE tariant; diff --git a/mysql-test/main/backup_server_restore.test b/mysql-test/main/backup_server_restore.test new file mode 100644 index 0000000000000..ef1f5af27a68b --- /dev/null +++ b/mysql-test/main/backup_server_restore.test @@ -0,0 +1,38 @@ +--source include/not_windows.inc +--source include/have_innodb.inc + +--echo Prepare database +CREATE TABLE tinno (i INTEGER) ENGINE=InnoDB; +INSERT INTO tinno VALUES (1), (2), (3), (4); +CREATE TABLE tariatr (i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO tariatr VALUES (2), (3), (5), (7); +CREATE TABLE tariant (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO tariant VALUES (1), (1), (2), (3), (5); + +--echo Back up the database +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +--echo Restore the database +--disable_query_log +call mtr.add_suppression("InnoDB: Did not find any checkpoint after LSN="); +call mtr.add_suppression("InnoDB: Renaming ib_[0-9]+.log to ib_logfile0"); +call mtr.add_suppression("mariadbd: Got error '145 \"Table was marked as crashed and should be repaired\"' for "); +call mtr.add_suppression("Checking table: "); +--enable_query_log +--let $restart_parameters=--datadir=$MYSQLTEST_VARDIR/some_directory +--source include/restart_mysqld.inc + +--echo Check contents after restore +SELECT * FROM tinno; +SELECT * FROM tariatr; +SELECT * FROM tariant; + +--echo Restart database in original data directory +--let $restart_parameters= +--source include/restart_mysqld.inc + +--echo Clean up +DROP TABLE tinno; +DROP TABLE tariatr; +DROP TABLE tariant; +--rmdir $MYSQLTEST_VARDIR/some_directory diff --git a/mysql-test/suite/backup/backup_innodb.test b/mysql-test/suite/backup/backup_innodb.test index 5f7bde34a31dd..d3ee82b1070aa 100644 --- a/mysql-test/suite/backup/backup_innodb.test +++ b/mysql-test/suite/backup/backup_innodb.test @@ -26,6 +26,7 @@ SET DEBUG_SYNC='now WAIT_FOR start'; INSERT INTO t(a) SELECT * FROM seq_1_to_30000; SET DEBUG_SYNC='now SIGNAL resume'; --connection backup +# FIXME: outside PMEM we may get ER_ERROR_ON_RENAME --reap } if (!$have_debug) { @@ -37,13 +38,16 @@ eval BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; --connection default ROLLBACK; SELECT * FROM t; +DELETE FROM t; let $datadir=`SELECT @@datadir`; ---source include/shutdown_mysqld.inc -#--rmdir $datadir -#--move_file $MYSQLTEST_VARDIR/some_directory $datadir ---rmdir $MYSQLTEST_VARDIR/some_directory ---source include/start_mysqld.inc +--let $restart_parameters=--datadir=$MYSQLTEST_VARDIR/some_directory +--source include/restart_mysqld.inc +SELECT * FROM t; +--let $restart_parameters= +--source include/restart_mysqld.inc SELECT * FROM t; DROP TABLE t; + +--rmdir $MYSQLTEST_VARDIR/some_directory diff --git a/storage/maria/CMakeLists.txt b/storage/maria/CMakeLists.txt index 9bdd729840077..7b4270ed4366d 100644 --- a/storage/maria/CMakeLists.txt +++ b/storage/maria/CMakeLists.txt @@ -45,6 +45,7 @@ SET(ARIA_SOURCES ma_init.c ma_open.c ma_extra.c ma_info.c ma_rkey.c ha_maria.h maria_def.h ma_recovery_util.c ma_servicethread.c ma_norec.c ma_crypt.c ma_backup.c + ma_backup.cc ma_backup.h ) IF(APPLE) diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 8f5e47daea728..624a7f0a043d3 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -23,6 +23,7 @@ #include #include #include "ha_maria.h" +#include "ma_backup.h" #include "trnman_public.h" #include "trnman.h" @@ -3942,6 +3943,9 @@ static int ha_maria_init(void *p) maria_hton->prepare_for_backup= maria_prepare_for_backup; maria_hton->end_backup= maria_end_backup; maria_hton->update_optimizer_costs= aria_update_optimizer_costs; + maria_hton->backup_start= aria_backup_start; + maria_hton->backup_step= aria_backup_step; + maria_hton->backup_end= aria_backup_end; /* TODO: decide if we support Maria being used for log tables */ maria_hton->flags= (HTON_CAN_RECREATE | HTON_SUPPORT_LOG_TABLES | diff --git a/storage/maria/ma_backup.cc b/storage/maria/ma_backup.cc new file mode 100644 index 0000000000000..9d3e0c1a3d82e --- /dev/null +++ b/storage/maria/ma_backup.cc @@ -0,0 +1,362 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "maria_def.h" +#include "ma_backup.h" +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#include +#include +#endif + +/* + Implementation of functions declatred in ma_backup.h: + BACKUP SERVER support for Aria engine +*/ + +using namespace std::string_literals; +namespace +{ + class Source_dir + { + public: + Source_dir(const char* path, myf flags) noexcept + { + dir_info= my_dir(path, flags); + if (!dir_info) + { + my_error(ER_CANT_READ_DIR, MYF(0), path, my_errno); + } + } + ~Source_dir() noexcept + { + my_dirend(dir_info); + } + bool is_error() const noexcept + { + return !dir_info; + } + template + int for_each(Fn fn) const noexcept + { + for (size_t i= 0; i < dir_info->number_of_files; i++) + { + if (fn(dir_info->dir_entry[i]) != 0) + return 1; + } + return 0; + } + + private: + MY_DIR *dir_info {nullptr}; + }; + + + /** Backup state; protected by log_sys.latch */ + class Aria_backup + { + public: + explicit Aria_backup(THD *thd, backup_target tgt) noexcept + : target(tgt) +#ifndef _WIN32 + , datadir_fd(open(maria_data_root, O_DIRECTORY)) + { + if (datadir_fd < 0) + { + my_error(ER_CANT_READ_DIR, MYF(0), maria_data_root, errno); + return; + } +#else + { +#endif // _WIN32 + translog_disable_purge(); + } + + bool is_initialized() const noexcept + { +#ifndef _WIN32 + return datadir_fd >= 0; +#else + return true; +#endif // _WIN32 + } + + ~Aria_backup() noexcept + { +#ifndef _WIN32 + if (datadir_fd >= 0) + close(datadir_fd); +#endif // _WIN32 + } + + int end(THD *thd, bool abort) noexcept + { + int ret_val = 0; + if (!abort) { + if (int err= perform_backup() != 0) + { + ret_val= err; + }; + } + translog_enable_purge(); + return ret_val; + } + private: + backup_target target; +#ifndef _WIN32 + const int datadir_fd; +#endif + static const std::vector data_exts; + static const std::string log_file_prefix; + using dir_name = std::string; + using dir_contents = std::vector; + using database_dir = std::pair; + std::vector database_dirs; + std::vector log_files; + bool have_control_file = false; + + int perform_backup() noexcept + { + if (scan_datadir()) + return 1; + if (copy_databases()) + return 1; + if (copy_control_file()) + return 1; + if(translog_flush(translog_get_horizon())) + return 1; + if (copy_logs()) + return 1; + return 0; + } + + int scan_datadir() noexcept + { + const char* base_dir = maria_data_root; + Source_dir datadir(base_dir, MYF(MY_WANT_STAT)); + if (datadir.is_error()) + return 1; + datadir.for_each([this](const fileinfo &fi) + { + if (fi.mystat->st_mode & S_IFDIR) + { + if (scan_database_dir(fi.name) != 0) + return 1; + } else if (begins_with(fi.name, log_file_prefix)) + log_files.emplace_back(fi.name); + else if (strcmp(fi.name, "aria_log_control") == 0) + have_control_file = true; + return 0; + }); + return 0; + } + + int scan_database_dir(const char* dir_name) noexcept + { + const char* base_dir = maria_data_root; + std::string dir_path = std::string(base_dir) + "/" + dir_name; + Source_dir db_dir(dir_path.c_str(), MYF(0)); + if (db_dir.is_error()) + return 1; + std::vector files_to_backup; + db_dir.for_each([&files_to_backup](const fileinfo &fi) + { + if (is_db_file(fi.name)) + files_to_backup.emplace_back(fi.name); + return 0; + }); + if (!files_to_backup.empty()) + database_dirs.emplace_back(dir_name, std::move(files_to_backup)); + return 0; + } + + int copy_databases() noexcept + { + for (const database_dir& dir : database_dirs) + { + const char* dir_name = dir.first.c_str(); + if (ensure_target_subdir(dir_name) != 0) + { + my_error(ER_CANT_CREATE_FILE, MYF(0), dir_name, errno); + return 1; + } + if (copy_database(dir) != 0) + return 1; + } + return 0; + } + + /* + Create directory in the target directory if it does not exist. + Return 0 on success, non-0 on failure. Set errno in case of failure + */ + int ensure_target_subdir(const char* name) noexcept + { +#ifdef _WIN32 + std::string dir_path= targetPath() + "/" + name; + if (!CreateDirectory(dir_path.c_str(), nullptr)) + { + DWORD err = GetLastError(); + if (err != ERROR_ALREADY_EXISTS) + { + my_osmaperr(err); + return 1; + } + } +#else + if (mkdirat(target.fd, name, 0777) != 0) + return (errno != EEXIST); +#endif + return 0; + } + + int copy_database(const database_dir& dir) noexcept + { + for (const std::string& file : dir.second) + { + std::string file_path= dir.first + "/" + file; + if (copy_file(file_path) != 0) + return 1; + } + return 0; + } + + int copy_control_file() noexcept + { + if (!have_control_file) + return 0; + return copy_file("aria_log_control"); + } + + int copy_logs() noexcept + { + for (const std::string& file : log_files) + { + if (copy_file(file) != 0) + return 1; + } + return 0; + } + + int copy_file(const std::string &path) const noexcept + { +#ifndef _WIN32 + int ret_val = 0; + int src_fd = openat(datadir_fd, path.c_str(), O_RDONLY); + if (src_fd < 0) + { + my_error(ER_CANT_OPEN_FILE, MYF(0), path.c_str(), errno); + return 1; + } + int tgt_fd = openat(target.fd, path.c_str(), + O_CREAT | O_EXCL | O_WRONLY, 0777); + if (tgt_fd < 0) + { + my_error(ER_CANT_CREATE_FILE, MYF(0), path.c_str(), errno); + ret_val = 1; + goto finish; + } + if (copy_entire_file(src_fd, tgt_fd) != 0) + { + my_error(ER_ERROR_ON_WRITE, MYF(0), path.c_str(), errno); + ret_val = 1; + } + close(tgt_fd); + finish: + close(src_fd); + return ret_val; +#else + std::string src_path= std::string(maria_data_root) + "/" + path; + std::string dest_path= targetPath() + "/" + path; + if(!CopyFileExA(src_path.c_str(), dest_path.c_str(), nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_CREATE_FILE, MYF(0), dest_path.c_str(), errno); + return 1; + } + return 0; +#endif + } + + + static bool is_db_file(const char* file_name) noexcept + { + for (const std::string& ext : data_exts) + { + if (ends_with(file_name, ext)) + return true; + } + /* As a stop-gap db/opt files are also copied here, this should be done in SQL layer. */ + return !strcmp(file_name, "db.opt"); + } + + static bool ends_with(const char* str, const std::string& suffix) noexcept + { + size_t str_len = strlen(str); + size_t suffix_len = suffix.size(); + if (str_len < suffix_len) + return false; + return memcmp(str + str_len - suffix_len, + suffix.data(), + suffix_len) == 0; + } + + static bool begins_with(const char* str, const std::string& prefix) noexcept + { + return strncmp(str, prefix.data(), prefix.size()) == 0; + } + +#ifdef _WIN32 + /** @return the target directory path */ + std::string targetPath() const + { + return std::string(target.path); + } +#endif + }; + + /* TODO: .frm failes are not Aria-specific, .MYD and .MYI are MyISAM files; + they are copied here as a stop-gap */ + const std::vector + Aria_backup::data_exts {".MAD"s, ".MAI"s, "MYD"s, "MYI"s, "frm"s}; + const std::string Aria_backup::log_file_prefix {"aria_log."}; + + std::unique_ptr aria_backup; +} + +int aria_backup_start(THD *thd, backup_target target) noexcept +{ + aria_backup= std::make_unique(thd, target); + return !aria_backup->is_initialized(); +} + +int aria_backup_step(THD *thd) noexcept +{ + return 0; +} + +int aria_backup_end(THD *thd, bool abort) noexcept +{ + int ret_val= aria_backup->end(thd, abort); + aria_backup.reset(); + return ret_val; +} diff --git a/storage/maria/ma_backup.h b/storage/maria/ma_backup.h new file mode 100644 index 0000000000000..3bec606648dac --- /dev/null +++ b/storage/maria/ma_backup.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#pragma once + +/* BACKUP SERVER support for Aria engine. */ + +#include +#include + +/** + Start of BACKUP SERVER: collect all files to be backed up + @param thd current session + @param target target directory + @return error code + @retval 0 on success +*/ +int aria_backup_start(THD *thd, backup_target target) noexcept; + +/** + Process a file that was collected in backup_start(). + @param thd current session + @return number of files remaining, or negative on error + @retval 0 on completion +*/ +int aria_backup_step(THD *thd) noexcept; + +/** + Finish copying and determine the logical time of the backup snapshot. + @param thd current session + @param abort whether BACKUP SERVER was aborted + @return error code + @retval 0 on success +*/ +int aria_backup_end(THD *thd, bool abort) noexcept; From c90f391acc33fe19f1f4c339448f019436b601ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jun 2026 07:32:05 +0300 Subject: [PATCH 03/25] fixup! c0e48fc966d3d3926bade68c7c5bfdb9fe88ad3a --- mysql-test/suite/backup/backup_innodb.result | 6 +++++- storage/innobase/handler/backup_innodb.cc | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/backup/backup_innodb.result b/mysql-test/suite/backup/backup_innodb.result index e1e856e93bae0..13f6202773e5f 100644 --- a/mysql-test/suite/backup/backup_innodb.result +++ b/mysql-test/suite/backup/backup_innodb.result @@ -17,8 +17,12 @@ ROLLBACK; SELECT * FROM t; a b 1 -# restart +DELETE FROM t; +# restart: --datadir=MYSQLTEST_VARDIR/some_directory SELECT * FROM t; a b 1 +# restart +SELECT * FROM t; +a b DROP TABLE t; diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index c97bcb42affdb..d7b0e87e625f7 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -542,7 +542,7 @@ class InnoDB_backup @param c offset of the FILE_CHECKPOINT mini-transaction @return error code @retval 0 on success */ - static int write_checkpoint(int dst, uint64_t c) noexcept + static int write_checkpoint(IF_WIN(HANDLE,int) dst, uint64_t c) noexcept { #ifdef _WIN32 using tpool::pwrite; From e2e57f2c81f9d691947634168a37bf3fdfe530d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jun 2026 11:35:32 +0300 Subject: [PATCH 04/25] Correctly copy the last one or two log files --- storage/innobase/handler/backup_innodb.cc | 84 +++++++++++++---------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index d7b0e87e625f7..a58a8de308371 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -224,7 +224,7 @@ class InnoDB_backup ctx.last_lsn= log_sys.get_flushed_lsn(std::memory_order_relaxed); while (!logs.empty()) { - const lsn_t lsn{logs.back()}; + lsn_t lsn{logs.back()}; if (lsn > ctx.last_lsn) break; if (lsn > ctx.max_first_lsn) @@ -238,21 +238,33 @@ class InnoDB_backup } { - lsn_t lsn= log_sys.get_first_lsn(); + lsn_t lsn{log_sys.get_first_lsn()}; if (lsn > ctx.max_first_lsn && lsn < ctx.last_lsn) { const lsn_t end_lsn{lsn + log_sys.capacity()}; ctx.max_first_lsn= lsn; log_sys.latch.wr_unlock(); + bool live_hardlink; if (UNIV_UNLIKELY(ctx.last_lsn > end_lsn)) { - /* Wait for checkpoint_complete(). */ + live_hardlink= true; + fail= link_or_move(lsn, &live_hardlink, ctx, target); + if (fail) + goto skip_log_dup; + /* Wait for checkpoint_complete(). If the previous link_or_move() + set live_hardlink, the file will be a read-only clone by now. */ buf_flush_sync_batch(end_lsn, true); - lsn= log_get_lsn(); - ut_ad(lsn >= end_lsn); + ut_ad(logs.size() == 1); + ut_ad(logs.back() == lsn); + logs.clear(); + lsn= log_sys.get_first_lsn(); + ut_ad(lsn == end_lsn); + ctx.max_first_lsn= lsn; + ctx.last_lsn= log_get_lsn(); + ut_ad(ctx.last_lsn >= end_lsn); } - bool live_hardlink= false; + live_hardlink= false; fail= link_or_move(lsn, &live_hardlink, ctx, target); log_sys.latch.wr_lock(); if (fail) @@ -268,18 +280,11 @@ class InnoDB_backup ut_ad(!log_sys.resize_in_progress()); ut_ad(log_sys.archive); - /* FIXME: If we made a hard link to the last log file, - we must retain and restore in fini() a copy of the - checkpoint header in the last log file of our backup. - - Otherwise, the server may overwrite the log header when - that file is still open for writing. The symptoms are as follows - (easy to reproduce by guaranteeing old_size!=0 below - (running the server with innodb_log_archiving=OFF): - - [ERROR] InnoDB: Did not find any checkpoint after LSN=12288 - [Warning] InnoDB: Renaming ib_0000000000003000.log to ib_logfile0 - */ + /* Note: If we temporarily made a hard link to the last log file + which is writeable by the server, fini() will copy the file. + If it is also the first (and only) log file in our backup, + write_checkpoint() will write a checkpoint header that identifies + the starting point of recovering the backup. */ if (old_size) { @@ -344,21 +349,24 @@ class InnoDB_backup my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), s, d, errno); fail= 1; } - /* Trim the file to the desired size after copying it. */ - HANDLE dh= CreateFile(d, GENERIC_READ, FILE_SHARE_READ, nullptr, - OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); - if (dh == INVALID_HANDLE_VALUE) - goto fail; - - LARGE_INTEGER li; - li.QuadPart= size; - BOOL ok= SetFilePointerEx(dh, li, nullptr, FILE_BEGIN) && - (ctx.max_first_lsn != ctx.first_lsn || - !write_checkpoint(dh, ctx.checkpoint_end_lsn - ctx.first_lsn + - log_sys.START_OFFSET)); - CloseHandle(dh); - if (!ok) - goto fail; + else + { + /* Trim the file to the desired size after copying it. */ + HANDLE dh= CreateFile(d, GENERIC_READ, FILE_SHARE_READ, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (dh == INVALID_HANDLE_VALUE) + goto fail; + + LARGE_INTEGER li; + li.QuadPart= size; + BOOL ok= SetFilePointerEx(dh, li, nullptr, FILE_BEGIN) && + (ctx.max_first_lsn != ctx.first_lsn || + !write_checkpoint(dh, ctx.checkpoint_end_lsn - ctx.first_lsn + + log_sys.START_OFFSET)); + CloseHandle(dh); + if (!ok) + goto fail; + } #else ut_ad(target.directory); int s= openat(target.fd, "ib_logfile101", O_RDONLY); @@ -599,7 +607,6 @@ class InnoDB_backup IF_WIN(const backup_target&,backup_target) target) noexcept { - ut_ad(!clone || !*clone); const std::string p{log_sys.get_archive_path(lsn)}; const char *const path= p.c_str(), *basename= strrchr(path, '/'); if (!basename) @@ -611,7 +618,7 @@ class InnoDB_backup #ifdef _WIN32 std::string b{target.path}; b.push_back('/'); - b.append(clone ? "ib_logfile101" : basename); + b.append((clone && *!clone) ? "ib_logfile101" : basename); const char *destname= b.c_str(); unsigned long err; @@ -652,9 +659,10 @@ class InnoDB_backup if (move ? !renameat(AT_FDCWD, path, target.fd, basename) : !linkat(AT_FDCWD, path, target.fd, - clone ? "ib_logfile101" : basename, AT_SYMLINK_FOLLOW)) + (clone && !*clone) ? "ib_logfile101" : basename, + AT_SYMLINK_FOLLOW)) { -#ifdef __linux__ +# ifdef __linux__ if (!move || lsn != ctx.first_lsn); else if (off_t garbage= (ctx.checkpoint - lsn) & ~4095ULL) /* Best effort to punch a hole to free up some garbage in @@ -668,7 +676,7 @@ class InnoDB_backup close(dst); fchmodat(target.fd, basename, 0444, 0); } -#endif +# endif if (clone) *clone= !move; return 0; From e096bd33c78baee68af6f7b951e13636e0b92fd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jun 2026 12:19:48 +0300 Subject: [PATCH 05/25] fixup! e2e57f2c81f9d691947634168a37bf3fdfe530d9 --- storage/innobase/handler/backup_innodb.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index a58a8de308371..f3c50418b4e31 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -618,7 +618,7 @@ class InnoDB_backup #ifdef _WIN32 std::string b{target.path}; b.push_back('/'); - b.append((clone && *!clone) ? "ib_logfile101" : basename); + b.append((clone && !*clone) ? "ib_logfile101" : basename); const char *destname= b.c_str(); unsigned long err; From 32cea9826cf2d24b8e89dd7005599815c0c125a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jun 2026 15:06:13 +0300 Subject: [PATCH 06/25] fixup! 7f207a291cb8b9903c08a05b0daa0a7eb90c3921 --- storage/innobase/handler/backup_innodb.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index f3c50418b4e31..6c473c47c9fdd 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -352,7 +352,8 @@ class InnoDB_backup else { /* Trim the file to the desired size after copying it. */ - HANDLE dh= CreateFile(d, GENERIC_READ, FILE_SHARE_READ, nullptr, + HANDLE dh= CreateFile(d, GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); if (dh == INVALID_HANDLE_VALUE) goto fail; From 56b8e279371a3a5194ae87e9575e32a0e6b5c743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 2 Jun 2026 09:17:57 +0300 Subject: [PATCH 07/25] Preserve the backup directory --- .../suite/backup/backup_innodb,debug.rdiff | 2 +- mysql-test/suite/backup/backup_innodb.result | 6 ++--- mysql-test/suite/backup/backup_innodb.test | 22 +++++++++++++------ 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/mysql-test/suite/backup/backup_innodb,debug.rdiff b/mysql-test/suite/backup/backup_innodb,debug.rdiff index 02d25801b75a7..e505f84d5f0d8 100644 --- a/mysql-test/suite/backup/backup_innodb,debug.rdiff +++ b/mysql-test/suite/backup/backup_innodb,debug.rdiff @@ -5,7 +5,7 @@ DELETE FROM t; connect backup,localhost,root; +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; - BACKUP SERVER TO 'MYSQLTEST_VARDIR/some_directory'; + BACKUP SERVER TO 'MYSQLTEST_VARDIR/backup_innodb_'; +connection default; +SET DEBUG_SYNC='now WAIT_FOR start'; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; diff --git a/mysql-test/suite/backup/backup_innodb.result b/mysql-test/suite/backup/backup_innodb.result index 13f6202773e5f..1a18521e662b0 100644 --- a/mysql-test/suite/backup/backup_innodb.result +++ b/mysql-test/suite/backup/backup_innodb.result @@ -2,7 +2,7 @@ CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) ENGINE=INNODB; BEGIN; INSERT INTO t SET a=1; -BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED'; ROLLBACK; SELECT * FROM t; a b @@ -10,7 +10,7 @@ a b BEGIN; DELETE FROM t; connect backup,localhost,root; -BACKUP SERVER TO 'MYSQLTEST_VARDIR/some_directory'; +BACKUP SERVER TO 'MYSQLTEST_VARDIR/backup_innodb_'; disconnect backup; connection default; ROLLBACK; @@ -18,7 +18,7 @@ SELECT * FROM t; a b 1 DELETE FROM t; -# restart: --datadir=MYSQLTEST_VARDIR/some_directory +# restart: with restart_parameters SELECT * FROM t; a b 1 diff --git a/mysql-test/suite/backup/backup_innodb.test b/mysql-test/suite/backup/backup_innodb.test index d3ee82b1070aa..915eff2a0010d 100644 --- a/mysql-test/suite/backup/backup_innodb.test +++ b/mysql-test/suite/backup/backup_innodb.test @@ -7,8 +7,13 @@ ENGINE=INNODB; BEGIN; INSERT INTO t SET a=1; -evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; ---rmdir $MYSQLTEST_VARDIR/some_directory +# In case this test is being run with --repeat +# clean up after a previous iteration of the test. +--error 0,1 +--rmdir $MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED + +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED'; +--rmdir $MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED ROLLBACK; # BACKUP SERVER will implicitly commit the current transaction SELECT * FROM t; @@ -20,7 +25,8 @@ DELETE FROM t; if ($have_debug) { SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR -send_eval BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--replace_regex /backup_innodb_\d*/backup_innodb_/ +send_eval BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED'; --connection default SET DEBUG_SYNC='now WAIT_FOR start'; INSERT INTO t(a) SELECT * FROM seq_1_to_30000; @@ -30,8 +36,8 @@ SET DEBUG_SYNC='now SIGNAL resume'; --reap } if (!$have_debug) { ---replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR -eval BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR $MTR_COMBINATION_ARCHIVED ARCHIVED +eval BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED'; } --disconnect backup @@ -41,7 +47,8 @@ SELECT * FROM t; DELETE FROM t; let $datadir=`SELECT @@datadir`; ---let $restart_parameters=--datadir=$MYSQLTEST_VARDIR/some_directory +--let $restart_noprint=1 +--let $restart_parameters=--datadir=$MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED --source include/restart_mysqld.inc SELECT * FROM t; @@ -50,4 +57,5 @@ SELECT * FROM t; SELECT * FROM t; DROP TABLE t; ---rmdir $MYSQLTEST_VARDIR/some_directory +# We will intentionally not remove the backup directory, so that it +# may be preserved for analysis in case the test case fails. From c310d7cf477e8db2b4911d69985fc237cba5133f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 2 Jun 2026 09:19:36 +0300 Subject: [PATCH 08/25] Fix up the checkpoint headers on Windows --- storage/innobase/handler/backup_innodb.cc | 30 +++++++++++++++++------ 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index 6c473c47c9fdd..2d219f64eb863 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -360,10 +360,12 @@ class InnoDB_backup LARGE_INTEGER li; li.QuadPart= size; - BOOL ok= SetFilePointerEx(dh, li, nullptr, FILE_BEGIN) && - (ctx.max_first_lsn != ctx.first_lsn || - !write_checkpoint(dh, ctx.checkpoint_end_lsn - ctx.first_lsn + - log_sys.START_OFFSET)); + BOOL ok= SetFilePointerEx(dh, li, nullptr, FILE_BEGIN); + if (ok && os_file_set_sparse_win32(dh)) + std::ignore= os_file_punch_hole(dh, 0, log_sys.START_OFFSET); + if (ok && ctx.max_first_lsn == ctx.first_lsn) + ok= !write_checkpoint(dh, ctx.checkpoint_end_lsn - ctx.first_lsn + + log_sys.START_OFFSET); CloseHandle(dh); if (!ok) goto fail; @@ -634,10 +636,24 @@ class InnoDB_backup my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), path, basename, errno); return -1; } -# if 0 + if (lsn < ctx.checkpoint) - /* FIXME: punch hole */; -# endif + { + HANDLE dh= CreateFile(destname, GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (dh == INVALID_HANDLE_VALUE) + goto fail; + if (os_file_set_sparse_win32(dh)) + std::ignore= + os_file_punch_hole(dh, 0, log_sys.START_OFFSET + + (((ctx.checkpoint - lsn) + 4095) & ~4095ULL)); + int fail= write_checkpoint(dh, ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + CloseHandle(dh); + if (fail) + goto fail; + } } else if (!CreateHardLink(destname, path, nullptr)) { From afbafab70608d8a31c52c3491e390e21ca3d2e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 2 Jun 2026 10:44:20 +0300 Subject: [PATCH 09/25] fixup! c310d7cf477e8db2b4911d69985fc237cba5133f --- storage/innobase/handler/backup_innodb.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index 2d219f64eb863..4e48bccc2081f 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -639,6 +639,8 @@ class InnoDB_backup if (lsn < ctx.checkpoint) { + if (!SetFileAttributes(destname, FILE_ATTRIBUTE_NORMAL)) + goto fail; HANDLE dh= CreateFile(destname, GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); @@ -647,7 +649,7 @@ class InnoDB_backup if (os_file_set_sparse_win32(dh)) std::ignore= os_file_punch_hole(dh, 0, log_sys.START_OFFSET + - (((ctx.checkpoint - lsn) + 4095) & ~4095ULL)); + ((ctx.checkpoint - lsn) & ~4095ULL)); int fail= write_checkpoint(dh, ctx.checkpoint_end_lsn - lsn + log_sys.START_OFFSET); CloseHandle(dh); From d803c8bb9b4ed92e244090302fb139aa380dca54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 2 Jun 2026 10:46:33 +0300 Subject: [PATCH 10/25] Revert "Preserve the backup directory" This reverts commit 56b8e279371a3a5194ae87e9575e32a0e6b5c743. --- .../suite/backup/backup_innodb,debug.rdiff | 2 +- mysql-test/suite/backup/backup_innodb.result | 6 ++--- mysql-test/suite/backup/backup_innodb.test | 22 ++++++------------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/mysql-test/suite/backup/backup_innodb,debug.rdiff b/mysql-test/suite/backup/backup_innodb,debug.rdiff index e505f84d5f0d8..02d25801b75a7 100644 --- a/mysql-test/suite/backup/backup_innodb,debug.rdiff +++ b/mysql-test/suite/backup/backup_innodb,debug.rdiff @@ -5,7 +5,7 @@ DELETE FROM t; connect backup,localhost,root; +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; - BACKUP SERVER TO 'MYSQLTEST_VARDIR/backup_innodb_'; + BACKUP SERVER TO 'MYSQLTEST_VARDIR/some_directory'; +connection default; +SET DEBUG_SYNC='now WAIT_FOR start'; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; diff --git a/mysql-test/suite/backup/backup_innodb.result b/mysql-test/suite/backup/backup_innodb.result index 1a18521e662b0..13f6202773e5f 100644 --- a/mysql-test/suite/backup/backup_innodb.result +++ b/mysql-test/suite/backup/backup_innodb.result @@ -2,7 +2,7 @@ CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) ENGINE=INNODB; BEGIN; INSERT INTO t SET a=1; -BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED'; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; ROLLBACK; SELECT * FROM t; a b @@ -10,7 +10,7 @@ a b BEGIN; DELETE FROM t; connect backup,localhost,root; -BACKUP SERVER TO 'MYSQLTEST_VARDIR/backup_innodb_'; +BACKUP SERVER TO 'MYSQLTEST_VARDIR/some_directory'; disconnect backup; connection default; ROLLBACK; @@ -18,7 +18,7 @@ SELECT * FROM t; a b 1 DELETE FROM t; -# restart: with restart_parameters +# restart: --datadir=MYSQLTEST_VARDIR/some_directory SELECT * FROM t; a b 1 diff --git a/mysql-test/suite/backup/backup_innodb.test b/mysql-test/suite/backup/backup_innodb.test index 915eff2a0010d..d3ee82b1070aa 100644 --- a/mysql-test/suite/backup/backup_innodb.test +++ b/mysql-test/suite/backup/backup_innodb.test @@ -7,13 +7,8 @@ ENGINE=INNODB; BEGIN; INSERT INTO t SET a=1; -# In case this test is being run with --repeat -# clean up after a previous iteration of the test. ---error 0,1 ---rmdir $MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED - -evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED'; ---rmdir $MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory ROLLBACK; # BACKUP SERVER will implicitly commit the current transaction SELECT * FROM t; @@ -25,8 +20,7 @@ DELETE FROM t; if ($have_debug) { SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR ---replace_regex /backup_innodb_\d*/backup_innodb_/ -send_eval BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED'; +send_eval BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; --connection default SET DEBUG_SYNC='now WAIT_FOR start'; INSERT INTO t(a) SELECT * FROM seq_1_to_30000; @@ -36,8 +30,8 @@ SET DEBUG_SYNC='now SIGNAL resume'; --reap } if (!$have_debug) { ---replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR $MTR_COMBINATION_ARCHIVED ARCHIVED -eval BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED'; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +eval BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; } --disconnect backup @@ -47,8 +41,7 @@ SELECT * FROM t; DELETE FROM t; let $datadir=`SELECT @@datadir`; ---let $restart_noprint=1 ---let $restart_parameters=--datadir=$MYSQLTEST_VARDIR/backup_innodb_$MTR_COMBINATION_ARCHIVED +--let $restart_parameters=--datadir=$MYSQLTEST_VARDIR/some_directory --source include/restart_mysqld.inc SELECT * FROM t; @@ -57,5 +50,4 @@ SELECT * FROM t; SELECT * FROM t; DROP TABLE t; -# We will intentionally not remove the backup directory, so that it -# may be preserved for analysis in case the test case fails. +--rmdir $MYSQLTEST_VARDIR/some_directory From 5633fc2bf7fd070fd482f60063fd41c3298d36ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 2 Jun 2026 16:06:42 +0300 Subject: [PATCH 11/25] Avoid CopyFileEx() when copying parts of InnoDB log --- sql/sql_backup.cc | 61 ++++++----- sql/sql_backup_interface.h | 29 +++--- storage/innobase/handler/backup_innodb.cc | 118 ++++++++++++++++------ 3 files changed, 137 insertions(+), 71 deletions(-) diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc index 224345c5d6ec6..6bbb44e9de03e 100644 --- a/sql/sql_backup.cc +++ b/sql/sql_backup.cc @@ -20,8 +20,12 @@ #include "sql_backup.h" #include "sql_backup_interface.h" #include "sql_parse.h" +#include "aligned.h" +#ifdef _WIN32 +# include "tpool.h" +#endif -#ifndef _WIN32 +#if defined __linux__ || defined __FreeBSD__ using copying_step= ssize_t(int,int,size_t,off_t*); template static ssize_t copy(int in_fd, int out_fd, off_t offset, off_t end) noexcept @@ -38,24 +42,25 @@ static ssize_t copy(int in_fd, int out_fd, off_t offset, off_t end) noexcept return -1; } } -# if defined __linux__ || defined __FreeBSD__ + /* Copy between files in a single (type of) file system */ static inline ssize_t copy_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept { return copy_file_range(in_fd, offset, out_fd, offset, count, 0); } -# define cfr(src,dst,start,end) copy(src, dst, start, end) -# endif -# ifdef __linux__ -# include +# define cfr(src,dst,start,end) copy(src, dst, start, end) +#endif +#ifdef __linux__ +# include /* Copy a file to a stream or to a regular file. */ static inline ssize_t send_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept { return sendfile(out_fd, in_fd, offset, count); } -# else +#else +# ifndef _WIN32 # include "aligned.h" # include /** Copy a file using a memory mapping. @@ -66,22 +71,21 @@ send_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept @return error code @retval 0 on success @retval 1 if a memory mapping failed */ -static ssize_t mmap_copy(int in_fd, int out_fd, off_t o, off_t end) +static ssize_t mmap_copy(int in_fd, int out_fd, uint64_t o, uint64_t end) { - assert(end >= o); #if SIZEOF_SIZE_T < 8 if (end != ssize_t(end)) return 1; #endif const size_t count= size_t(end - o); - void *p= mmap(nullptr, count, PROT_READ, MAP_SHARED, in_fd, o); + void *p= mmap(nullptr, count, PROT_READ, MAP_SHARED, in_fd, off_t(o)); if (p == MAP_FAILED) return 1; ssize_t ret; size_t c{count}; - for (const char *b= static_cast(p);; b+= ret, o+= ret) + for (const char *b= static_cast(p);; b+= ret, o+= uint64_t(ret)) { - ret= pwrite(out_fd, b, std::min(c, size_t(INT_MAX >> 20 << 20)), o); + ret= pwrite(out_fd, b, std::min(c, size_t(INT_MAX >> 20 << 20)), off_t(o)); if (ret < 0) break; c-= ret; @@ -99,6 +103,7 @@ static ssize_t mmap_copy(int in_fd, int out_fd, off_t o, off_t end) munmap(p, c); return ret; } +# endif /** Copy a file using positioned reads and writes. @param in_fd source file @@ -108,23 +113,27 @@ static ssize_t mmap_copy(int in_fd, int out_fd, off_t o, off_t end) @return error code @retval 0 on success @retval 1 if a memory mapping failed */ -static ssize_t pread_pwrite(int in_fd, int out_fd, off_t o, off_t end) +static ssize_t pread_pwrite(IF_WIN(HANDLE,int) in_fd, IF_WIN(HANDLE,int) out_fd, + uint64_t o, uint64_t end) noexcept { - assert(end >= o); +#ifdef _WIN32 + using tpool::pread; + using tpool::pwrite; +#endif constexpr size_t READ_WRITE_SIZE= 65536; char *b= static_cast(aligned_malloc(READ_WRITE_SIZE, 4096)); if (!b) return -1; ssize_t ret; - for (off_t count{end - o};; o+= ret) + for (uint64_t count{end - o};; o+= ret) { - ret= pread(in_fd, b, ssize_t(std::min(count, off_t{READ_WRITE_SIZE})), o); + ret= pread(in_fd, b, ssize_t(std::min(count, READ_WRITE_SIZE)), o); if (ret > 0) ret= pwrite(out_fd, b, ret, o); if (ret < 0) break; - count-= ret; + count-= uint64_t(ret); if (!count) { ret= 0; @@ -139,10 +148,12 @@ static ssize_t pread_pwrite(int in_fd, int out_fd, off_t o, off_t end) aligned_free(b); return ret; } -# endif +#endif #ifdef __APPLE__ /* The inline copy_entire_file() invokes fcopyfile() */ +#elif defined _WIN32 +/* CopyFileEx() should be used */ #else /** Copy a file (whole content). @param src source file descriptor @@ -155,36 +166,38 @@ extern "C" int copy_entire_file(int src, int dst) } #endif -/** Copy a file. +/** Copy a portion of a file. @param src source file descriptor @param dst target to append src to @param start first offset to copy @param end last offset to copy (exclusive) @return error code (negative) @retval 0 on success */ -extern "C" int copy_file(int src, int dst, off_t start, off_t end) +extern "C" int copy_file(IF_WIN(HANDLE,int) src, IF_WIN(HANDLE,int) dst, + uint64_t start, uint64_t end) { assert(end >= start); ssize_t ret; # ifdef cfr - if (!(ret= cfr(src, dst, start, end))) + if (!(ret= cfr(src, dst, off_t(start), off_t(end)))) return int(ret); # ifdef __linux__ if (errno == EOPNOTSUPP || errno == EXDEV) # endif # endif # ifdef __linux__ // starting with Linux 2.6.33, we can rely on sendfile(2) - ret= (start != 0 && start != lseek(dst, start, SEEK_SET)) + ret= (start != 0 && off_t(start) != lseek(dst, start, SEEK_SET)) ? -1 - : copy(src, dst, start, end); + : copy(src, dst, off_t(start), off_t(end)); # else +# ifndef _WIN32 if ((ret= mmap_copy(src, dst, start, end)) == 1) +# endif ret= pread_pwrite(src, dst, start, end); # endif assert(ret <= 0); return int(ret); } -#endif static my_bool backup_start(THD *thd, plugin_ref plugin, void *dst) noexcept { diff --git a/sql/sql_backup_interface.h b/sql/sql_backup_interface.h index a86a16ab505fc..e95ed41ade2b9 100644 --- a/sql/sql_backup_interface.h +++ b/sql/sql_backup_interface.h @@ -14,14 +14,13 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ #ifdef _WIN32 -/* You have to use CopyFileEx() and friends manually */ -#else -# if defined __APPLE__ +/* Use CopyFileEx() to copy entire files */ +#elif defined __APPLE__ /* You should invoke fclonefileat(2) manually before attempting copy_entire_file() or copy_file() */ -# include -# include -# include +# include +# include +# include /** Copy an entire file. @param src source file descriptor @param dst target to append src to @@ -31,27 +30,27 @@ inline int copy_entire_file(int src, int dst) { return fcopyfile(src, dst, NULL, COPYFILE_ALL | COPYFILE_CLONE); } -# else -# ifdef __cplusplus +#else +# ifdef __cplusplus extern "C" -# endif +# endif /** Copy an entire file. @param src source file descriptor @param dst target to append src to @return error code (negative) @retval 0 on success */ int copy_entire_file(int src, int dst); -# endif +#endif -# ifdef __cplusplus +#ifdef __cplusplus extern "C" -# endif -/** Copy a file. +#endif +/** Copy a portion of a file. @param src source file descriptor @param dst target to append src to @param start first offset to copy @param end last offset to copy (exclusive) @return error code (negative) @retval 0 on success */ -int copy_file(int src, int dst, off_t start, off_t end); -#endif +int copy_file(IF_WIN(HANDLE,int) src, IF_WIN(HANDLE,int) dst, + uint64_t start, uint64_t end); diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index 4e48bccc2081f..f5c5ebb72c64c 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -327,7 +327,7 @@ class InnoDB_backup const backup_context &ctx{trx->lock.backup}; if (ctx.max_first_lsn) { - const lsn_t size= + const uint64_t size= std::max(log_sys.FILE_SIZE_MIN, log_sys.START_OFFSET + (((ctx.last_lsn - ctx.max_first_lsn) + 4095) & ~4095ULL)); @@ -338,37 +338,53 @@ class InnoDB_backup std::string dst{src}; src.append("ib_logfile101"); log_sys.append_archive_name(dst, ctx.max_first_lsn); - const char *s= src.c_str(), *d= dst.c_str(); - if (!CopyFileEx(s, d, nullptr, nullptr, nullptr, - COPY_FILE_NO_BUFFERING) || - !SetFileAttributes(s, FILE_ATTRIBUTE_NORMAL) || - !DeleteFile(s)) + const char *s_= src.c_str(), *d_= dst.c_str(); + HANDLE s, d; + for (;;) + { + s= CreateFile(s_, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (s != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), s_, errno); + fail= 1; + goto done; + } + d= CreateFile(d_, GENERIC_WRITE, 0, my_win_file_secattr(), + CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); + if (d == INVALID_HANDLE_VALUE) { fail: - my_osmaperr(GetLastError()); - my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), s, d, errno); fail= 1; + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), s_, d_, errno); + CloseHandle(s); } else { - /* Trim the file to the desired size after copying it. */ - HANDLE dh= CreateFile(d, GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, - OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); - if (dh == INVALID_HANDLE_VALUE) + fail= copy_file(s, d, log_sys.START_OFFSET, size) || + (ctx.max_first_lsn == ctx.first_lsn && + write_checkpoint(d, ctx.checkpoint_end_lsn - ctx.first_lsn + + log_sys.START_OFFSET)); + if (!CloseHandle(d) || fail) goto fail; - LARGE_INTEGER li; - li.QuadPart= size; - BOOL ok= SetFilePointerEx(dh, li, nullptr, FILE_BEGIN); - if (ok && os_file_set_sparse_win32(dh)) - std::ignore= os_file_punch_hole(dh, 0, log_sys.START_OFFSET); - if (ok && ctx.max_first_lsn == ctx.first_lsn) - ok= !write_checkpoint(dh, ctx.checkpoint_end_lsn - ctx.first_lsn + - log_sys.START_OFFSET); - CloseHandle(dh); - if (!ok) - goto fail; + CloseHandle(s); + + if (!DeleteFile(s_)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_DELETE_FILE, MYF(ME_ERROR_LOG), s_, errno); + fail= 1; + } } #else ut_ad(target.directory); @@ -410,8 +426,8 @@ class InnoDB_backup } std::ignore= close(s); } - done:; #endif + done: /* TODO: punch hole to the start of the first log file if we had old_size!=0 */ sql_print_information("innodb_log_recovery_start=" LSN_PF @@ -467,6 +483,7 @@ class InnoDB_backup path.push_back('/'); backup_start(node->space); path.append(node->name); + /* FIXME: copy page ranges with copy_file() like everywhere else */ bool ok= CopyFileEx(node->name, path.c_str(), nullptr, nullptr, nullptr, COPY_FILE_NO_BUFFERING); backup_stop(node->space); @@ -576,7 +593,6 @@ class InnoDB_backup return 0; } -#ifndef _WIN32 /** Copy a log file. @param src source file @param dst target file @@ -586,7 +602,8 @@ class InnoDB_backup @param c offset of the FILE_CHECKPOINT mini-transaction @return error code @retval 0 on success */ - static int copy_log_file_part(int src, int dst, off_t start, off_t size, + static int copy_log_file_part(IF_WIN(HANDLE,int) src, IF_WIN(HANDLE,int) dst, + uint64_t start, uint64_t size, const backup_context &ctx, uint64_t c) noexcept { @@ -594,7 +611,6 @@ class InnoDB_backup return err; return write_checkpoint(dst, c); } -#endif /** Hard-link (copy) or rename (move) an archive log file. @param lsn The first LSN in the file @@ -667,9 +683,47 @@ class InnoDB_backup b.append(basename); destname= b.c_str(); - if (!CopyFileEx(path, destname, nullptr, nullptr, nullptr, - COPY_FILE_NO_BUFFERING)) - goto fail; + if (lsn >= ctx.checkpoint) + { + if (!CopyFileEx(path, destname, nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING)) + goto fail; + } + else + { + HANDLE s, d; + for (;;) + { + s= CreateFile(path, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (s != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + goto fail; + } + d= CreateFile(destname, GENERIC_WRITE, 0, my_win_file_secattr(), + CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); + if (d == INVALID_HANDLE_VALUE) + { + CloseHandle(s); + goto fail; + } + + int err= copy_log_file_part(s, d, log_sys.START_OFFSET + + (((ctx.checkpoint - lsn) + 4095) & ~4095ULL), + ctx.first_size, ctx, + ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + if (err | !(CloseHandle(s) & CloseHandle(d))) + goto fail; + } } else if (clone) *clone= true; @@ -730,7 +784,7 @@ class InnoDB_backup ctx.checkpoint_end_lsn - lsn + log_sys.START_OFFSET); - if ((close(dst) | close(src)) || err) + if (err | close(dst) | close(src)) goto fail; } #endif From 180f79d9591770632af097415c71db9b1c2b94c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 3 Jun 2026 08:57:16 +0300 Subject: [PATCH 12/25] fixup! 5633fc2bf7fd070fd482f60063fd41c3298d36ac --- sql/sql_backup.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc index 6bbb44e9de03e..c020dd116d372 100644 --- a/sql/sql_backup.cc +++ b/sql/sql_backup.cc @@ -20,8 +20,8 @@ #include "sql_backup.h" #include "sql_backup_interface.h" #include "sql_parse.h" -#include "aligned.h" #ifdef _WIN32 +# include "aligned.h" # include "tpool.h" #endif @@ -128,7 +128,8 @@ static ssize_t pread_pwrite(IF_WIN(HANDLE,int) in_fd, IF_WIN(HANDLE,int) out_fd, ssize_t ret; for (uint64_t count{end - o};; o+= ret) { - ret= pread(in_fd, b, ssize_t(std::min(count, READ_WRITE_SIZE)), o); + ret= pread(in_fd, b, + ssize_t(std::min(count, READ_WRITE_SIZE)), o); if (ret > 0) ret= pwrite(out_fd, b, ret, o); if (ret < 0) From 5d04c41c3f353c7369fb1b2f16626b587120e31f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 3 Jun 2026 08:57:47 +0300 Subject: [PATCH 13/25] fixup! 5633fc2bf7fd070fd482f60063fd41c3298d36ac Get a stack trace from Windows --- storage/innobase/os/os0file.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 4ef86095f78a3..b82ea77203624 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -49,6 +49,7 @@ Created 10/21/1995 Heikki Tuuri #include "fil0fil.h" #include "fsp0fsp.h" #include "buf0dblwr.h" +#include "log0recv.h" #include @@ -1825,6 +1826,7 @@ ulint os_file_get_last_error(bool report_all_errors, bool on_error_silent) " the directory. It may also be" " you have created a subdirectory" " of the same name as a data file."; + ut_a(!recv_recovery_is_on()); break; case ERROR_SHARING_VIOLATION: case ERROR_LOCK_VIOLATION: From da86ec2fc00f9c24677f7018069f1f62e6fc9f7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 3 Jun 2026 14:16:46 +0300 Subject: [PATCH 14/25] Use copy_file() for copying data files on Windows --- storage/innobase/handler/backup_innodb.cc | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index f5c5ebb72c64c..5bd55efe5cd39 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -481,13 +481,11 @@ class InnoDB_backup #ifdef _WIN32 std::string path{target.path}; path.push_back('/'); - backup_start(node->space); path.append(node->name); - /* FIXME: copy page ranges with copy_file() like everywhere else */ - bool ok= CopyFileEx(node->name, path.c_str(), nullptr, nullptr, nullptr, - COPY_FILE_NO_BUFFERING); - backup_stop(node->space); - if (!ok) + HANDLE f= CreateFile(path.c_str(), GENERIC_WRITE, 0, + my_win_file_secattr(), CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (f == INVALID_HANDLE_VALUE) { unsigned long err= GetLastError(); if (err == ERROR_PATH_NOT_FOUND && !tried_mkdir && @@ -505,7 +503,6 @@ class InnoDB_backup my_osmaperr(err); goto fail; } - break; #else int f; ut_ad(target.directory); @@ -548,16 +545,15 @@ class InnoDB_backup } goto fail; } - +#endif /* TODO: page range locking; avoid copying freed page ranges */ backup_start(node->space); int err= copy_file(node->handle, f, 0, off_t{node->size} * node->space->physical_size()); backup_stop(node->space); - if (close(f) || err) + if (IF_WIN(!CloseHandle(f), close(f)) || err) goto fail; break; -#endif } return 0; fail: @@ -685,6 +681,7 @@ class InnoDB_backup if (lsn >= ctx.checkpoint) { + sql_print_information("CopyFileEx %s, %s", path, destname); if (!CopyFileEx(path, destname, nullptr, nullptr, nullptr, COPY_FILE_NO_BUFFERING)) goto fail; From bcdcf854ea15aaf41d4c7de7d6e00f6389451c6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 3 Jun 2026 16:29:22 +0300 Subject: [PATCH 15/25] Implement page range locking --- storage/innobase/buf/buf0flu.cc | 32 +++++++++++++-------- storage/innobase/handler/backup_innodb.cc | 34 +++++++++++++++-------- storage/innobase/include/fil0fil.h | 3 ++ 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 6ebad0d869aa1..cd3fc58c830b4 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1398,7 +1398,9 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, flush: if (UNIV_UNLIKELY(to_withdraw != 0)) to_withdraw= buf_flush_LRU_to_withdraw(to_withdraw, *bpage); - if (bpage->id().page_no() < backup_page_end) + const uint32_t page{bpage->id().page_no()}; + if (page < backup_page_end && + page >= backup_page_end - space->BACKUP_BATCH_SIZE) bpage->lock.u_unlock(true); else if (bpage->flush(space)) { @@ -1565,7 +1567,9 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept space->is_rotational()) count+= buf_flush_try_neighbors(space, page_id, bpage, neighbors == 1, count, max_n); - else if (bpage->id().page_no() < backup_page_end) + else if (page_id.page_no() < backup_page_end && + page_id.page_no() >= + backup_page_end - space->BACKUP_BATCH_SIZE) { bpage->lock.u_unlock(true); continue; @@ -1724,17 +1728,23 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept } mysql_mutex_unlock(&buf_pool.flush_list_mutex); + uint32_t page, backup_page_end; - if (UNIV_UNLIKELY(space->writing_start()) && - bpage->id().page_no() < space->backup_page_end()) + if (UNIV_UNLIKELY(space->writing_start())) { - bpage->lock.u_unlock(true); - space->writing_stop(); - skip: - mysql_mutex_lock(&buf_pool.mutex); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - may_have_skipped= true; - goto done; + page= bpage->id().page_no(); + backup_page_end= space->backup_page_end(); + if (page < backup_page_end && + page >= backup_page_end - space->BACKUP_BATCH_SIZE) + { + bpage->lock.u_unlock(true); + space->writing_stop(); + skip: + mysql_mutex_lock(&buf_pool.mutex); + mysql_mutex_lock(&buf_pool.flush_list_mutex); + may_have_skipped= true; + goto done; + } } const bool written{bpage->flush(space)}; diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index 5bd55efe5cd39..03388e09078bf 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -179,9 +179,10 @@ class InnoDB_backup else if (fil_space_t *space= fil_space_t::get(id)) { int res= -1; + uint32_t start{0}; for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain); node; - node= UT_LIST_GET_NEXT(chain, node)) - if ((res= backup(node))) + start+= node->size, node= UT_LIST_GET_NEXT(chain, node)) + if ((res= backup(node, start))) break; space->release(); if (res) @@ -452,10 +453,11 @@ class InnoDB_backup } private: - /** Safely start backing up a tablespace file */ - static void backup_start(fil_space_t *space) noexcept + /** Safely start backing up a tablespace file + @param end last page to copy */ + static void backup_start(fil_space_t *space, uint32_t end) noexcept { - if (space->backup_start(space->size)) + if (space->backup_start(end)) os_aio_wait_until_no_pending_writes(false); } /* Stop backing up a tablespace */ @@ -473,8 +475,9 @@ class InnoDB_backup /** Back up a persistent InnoDB data file. @param node InnoDB data file + @param start first page number */ - int backup(fil_node_t *node) noexcept + int backup(fil_node_t *node, uint32_t start) noexcept { for (bool tried_mkdir{false};;) { @@ -546,11 +549,20 @@ class InnoDB_backup goto fail; } #endif - /* TODO: page range locking; avoid copying freed page ranges */ - backup_start(node->space); - int err= copy_file(node->handle, f, 0, - off_t{node->size} * node->space->physical_size()); - backup_stop(node->space); + int err{0}; + for (const uint32_t file_size{node->size}, + page_size{node->space->physical_size()};;) + { + const uint32_t end{start + fil_space_t::BACKUP_BATCH_SIZE}; + backup_start(node->space, end); + /* TODO: avoid copying freed page ranges */ + err= copy_file(node->handle, f, start, std::min(end, file_size) * + uint64_t{page_size}); + backup_stop(node->space); + if (err || (start= end) >= file_size) + break; + } + if (IF_WIN(!CloseHandle(f), close(f)) || err) goto fail; break; diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 06fb28dd98331..6a6680e667ae6 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1088,6 +1088,9 @@ struct fil_space_t final uint32_t backup_page_end() const noexcept { return backup_end.load(std::memory_order_relaxed); } + /** The size of a backup copy_file() batch in pages */ + static constexpr uint32_t BACKUP_BATCH_SIZE{64}; + /** Update the data structures on write completion */ void complete_write() noexcept; From 30cc18d43a8f84cdb80298ef067dc886d1f816a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 3 Jun 2026 18:08:59 +0300 Subject: [PATCH 16/25] fixup! bcdcf854ea15aaf41d4c7de7d6e00f6389451c6f --- storage/innobase/handler/backup_innodb.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index 03388e09078bf..d8f2b76d30e80 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -510,7 +510,9 @@ class InnoDB_backup int f; ut_ad(target.directory); # ifdef __APPLE__ - backup_start(node->space); + backup_start(node->space, + (node->space->size + fil_space_t::BACKUP_BATCH_SIZE - 1) & + ~fil_space_t::BACKUP_BATCH_SIZE); f= fclonefileat(node->handle, target.fd, node->name, 0); backup_stop(node->space); if (!f) From 9a7b3cd13bca6ae5b38408d9dfdb3908e6f2108d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 3 Jun 2026 18:21:37 +0300 Subject: [PATCH 17/25] fixup! da86ec2fc00f9c24677f7018069f1f62e6fc9f7d --- sql/sql_backup.cc | 7 ++++--- sql/sql_backup_interface.h | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc index c020dd116d372..0025fc03cc9ff 100644 --- a/sql/sql_backup.cc +++ b/sql/sql_backup.cc @@ -22,7 +22,6 @@ #include "sql_parse.h" #ifdef _WIN32 # include "aligned.h" -# include "tpool.h" #endif #if defined __linux__ || defined __FreeBSD__ @@ -113,7 +112,8 @@ static ssize_t mmap_copy(int in_fd, int out_fd, uint64_t o, uint64_t end) @return error code @retval 0 on success @retval 1 if a memory mapping failed */ -static ssize_t pread_pwrite(IF_WIN(HANDLE,int) in_fd, IF_WIN(HANDLE,int) out_fd, +static ssize_t pread_pwrite(IF_WIN(const native_file_handle&,int) in_fd, + IF_WIN(const native_file_handle&,int) out_fd, uint64_t o, uint64_t end) noexcept { @@ -174,7 +174,8 @@ extern "C" int copy_entire_file(int src, int dst) @param end last offset to copy (exclusive) @return error code (negative) @retval 0 on success */ -extern "C" int copy_file(IF_WIN(HANDLE,int) src, IF_WIN(HANDLE,int) dst, +extern "C" int copy_file(IF_WIN(const native_file_handle&,int) src, + IF_WIN(const native_file_handle&,int) dst, uint64_t start, uint64_t end) { assert(end >= start); diff --git a/sql/sql_backup_interface.h b/sql/sql_backup_interface.h index e95ed41ade2b9..3696803ca3f06 100644 --- a/sql/sql_backup_interface.h +++ b/sql/sql_backup_interface.h @@ -15,6 +15,7 @@ #ifdef _WIN32 /* Use CopyFileEx() to copy entire files */ +# include "tpool.h" #elif defined __APPLE__ /* You should invoke fclonefileat(2) manually before attempting copy_entire_file() or copy_file() */ @@ -52,5 +53,6 @@ extern "C" @param end last offset to copy (exclusive) @return error code (negative) @retval 0 on success */ -int copy_file(IF_WIN(HANDLE,int) src, IF_WIN(HANDLE,int) dst, +int copy_file(IF_WIN(const native_file_handle&,int) src, + IF_WIN(const native_file_handle&,int) dst, uint64_t start, uint64_t end); From 2723b136356ff9940d095e6a209fdfb5bc183734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 3 Jun 2026 19:06:48 +0300 Subject: [PATCH 18/25] fixup! 9a7b3cd13bca6ae5b38408d9dfdb3908e6f2108d --- sql/sql_backup.cc | 1 + sql/sql_backup_interface.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc index 0025fc03cc9ff..56d76aa10d0a7 100644 --- a/sql/sql_backup.cc +++ b/sql/sql_backup.cc @@ -22,6 +22,7 @@ #include "sql_parse.h" #ifdef _WIN32 # include "aligned.h" +# include "tpool.h" #endif #if defined __linux__ || defined __FreeBSD__ diff --git a/sql/sql_backup_interface.h b/sql/sql_backup_interface.h index 3696803ca3f06..516f9d3ea9a95 100644 --- a/sql/sql_backup_interface.h +++ b/sql/sql_backup_interface.h @@ -15,7 +15,7 @@ #ifdef _WIN32 /* Use CopyFileEx() to copy entire files */ -# include "tpool.h" +struct native_file_handle; #elif defined __APPLE__ /* You should invoke fclonefileat(2) manually before attempting copy_entire_file() or copy_file() */ From e058dd9240c98eefeff2bc71c91e8e16c61adfc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 4 Jun 2026 07:18:17 +0300 Subject: [PATCH 19/25] fixup! bcdcf854ea15aaf41d4c7de7d6e00f6389451c6f --- storage/innobase/handler/backup_innodb.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index d8f2b76d30e80..27369564c1a27 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -558,8 +558,8 @@ class InnoDB_backup const uint32_t end{start + fil_space_t::BACKUP_BATCH_SIZE}; backup_start(node->space, end); /* TODO: avoid copying freed page ranges */ - err= copy_file(node->handle, f, start, std::min(end, file_size) * - uint64_t{page_size}); + err= copy_file(node->handle, f, start * uint64_t{page_size}, + std::min(end, file_size) * uint64_t{page_size}); backup_stop(node->space); if (err || (start= end) >= file_size) break; From f2b8d0f8b222d1b439243095f4bd5b1452c81ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 4 Jun 2026 15:03:27 +0300 Subject: [PATCH 20/25] Create and use backup.cnf; do not copy too much log --- .../suite/backup/backup_innodb,debug.rdiff | 2 +- mysql-test/suite/backup/backup_innodb.result | 8 +- mysql-test/suite/backup/backup_innodb.test | 42 ++++- sql/sql_backup.cc | 62 ++++++- sql/sql_backup_interface.h | 17 +- storage/innobase/handler/backup_innodb.cc | 170 ++++++++++++------ 6 files changed, 230 insertions(+), 71 deletions(-) diff --git a/mysql-test/suite/backup/backup_innodb,debug.rdiff b/mysql-test/suite/backup/backup_innodb,debug.rdiff index 02d25801b75a7..ed165bdf9e59e 100644 --- a/mysql-test/suite/backup/backup_innodb,debug.rdiff +++ b/mysql-test/suite/backup/backup_innodb,debug.rdiff @@ -5,7 +5,7 @@ DELETE FROM t; connect backup,localhost,root; +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; - BACKUP SERVER TO 'MYSQLTEST_VARDIR/some_directory'; + BACKUP SERVER TO 'target_directory'; +connection default; +SET DEBUG_SYNC='now WAIT_FOR start'; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; diff --git a/mysql-test/suite/backup/backup_innodb.result b/mysql-test/suite/backup/backup_innodb.result index 13f6202773e5f..7ab700975ac78 100644 --- a/mysql-test/suite/backup/backup_innodb.result +++ b/mysql-test/suite/backup/backup_innodb.result @@ -2,7 +2,7 @@ CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) ENGINE=INNODB; BEGIN; INSERT INTO t SET a=1; -BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +BACKUP SERVER TO '$target_directory'; ROLLBACK; SELECT * FROM t; a b @@ -10,7 +10,7 @@ a b BEGIN; DELETE FROM t; connect backup,localhost,root; -BACKUP SERVER TO 'MYSQLTEST_VARDIR/some_directory'; +BACKUP SERVER TO 'target_directory'; disconnect backup; connection default; ROLLBACK; @@ -18,10 +18,12 @@ SELECT * FROM t; a b 1 DELETE FROM t; -# restart: --datadir=MYSQLTEST_VARDIR/some_directory +# restart: --defaults-file=MYSQLTEST_VARDIR/some_directory/backup.cnf --datadir=MYSQLTEST_VARDIR/some_directory SELECT * FROM t; a b 1 +DELETE FROM t; +ERROR HY000: Table 't' is read only # restart SELECT * FROM t; a b diff --git a/mysql-test/suite/backup/backup_innodb.test b/mysql-test/suite/backup/backup_innodb.test index d3ee82b1070aa..769c256f8c3d0 100644 --- a/mysql-test/suite/backup/backup_innodb.test +++ b/mysql-test/suite/backup/backup_innodb.test @@ -7,8 +7,17 @@ ENGINE=INNODB; BEGIN; INSERT INTO t SET a=1; -evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; ---rmdir $MYSQLTEST_VARDIR/some_directory +--let $target_directory=/tmp/some_directory$MTR_COMBINATION_ARCHIVED +# comment out the following line (and some more, including rmdir), +# to test cross-filesystem copy +--let $target_directory=$MYSQLTEST_VARDIR/some_directory + +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory + +evalp BACKUP SERVER TO '$target_directory'; +--rmdir $target_directory ROLLBACK; # BACKUP SERVER will implicitly commit the current transaction SELECT * FROM t; @@ -19,8 +28,8 @@ DELETE FROM t; --connect backup,localhost,root if ($have_debug) { SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; ---replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR -send_eval BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--replace_result $target_directory target_directory +send_eval BACKUP SERVER TO '$target_directory'; --connection default SET DEBUG_SYNC='now WAIT_FOR start'; INSERT INTO t(a) SELECT * FROM seq_1_to_30000; @@ -30,8 +39,8 @@ SET DEBUG_SYNC='now SIGNAL resume'; --reap } if (!$have_debug) { ---replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR -eval BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--replace_result $target_directory target_directory +eval BACKUP SERVER TO '$target_directory'; } --disconnect backup @@ -40,14 +49,29 @@ ROLLBACK; SELECT * FROM t; DELETE FROM t; -let $datadir=`SELECT @@datadir`; ---let $restart_parameters=--datadir=$MYSQLTEST_VARDIR/some_directory +if ($MARIADB_UPGRADE_EXE) { +let target_directory=$target_directory; +perl; +open IN, "<", "$ENV{MYSQLTEST_VARDIR}/my.cnf"; +open OUT, ">>", "$ENV{target_directory}/backup.cnf"; +print OUT while (); +close(IN); +close(OUT); +EOF +} +if (!$MARIADB_UPGRADE_EXE) { + --exec cat $MYSQLTEST_VARDIR/my.cnf >> $target_directory/backup.cnf +} +--let $restart_parameters=--defaults-file=$target_directory/backup.cnf --datadir=$target_directory --source include/restart_mysqld.inc SELECT * FROM t; +# we must have started up with nonzero innodb_log_recovery_target +--error ER_OPEN_AS_READONLY +DELETE FROM t; --let $restart_parameters= --source include/restart_mysqld.inc SELECT * FROM t; DROP TABLE t; ---rmdir $MYSQLTEST_VARDIR/some_directory +--rmdir $target_directory diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc index 56d76aa10d0a7..607116e725441 100644 --- a/sql/sql_backup.cc +++ b/sql/sql_backup.cc @@ -160,7 +160,7 @@ static ssize_t pread_pwrite(IF_WIN(const native_file_handle&,int) in_fd, /** Copy a file (whole content). @param src source file descriptor @param dst target to append src to -@return error code (negative) +@return error code (non-positive) @retval 0 on success */ extern "C" int copy_entire_file(int src, int dst) { @@ -173,7 +173,7 @@ extern "C" int copy_entire_file(int src, int dst) @param dst target to append src to @param start first offset to copy @param end last offset to copy (exclusive) -@return error code (negative) +@return error code (non-positive) @retval 0 on success */ extern "C" int copy_file(IF_WIN(const native_file_handle&,int) src, IF_WIN(const native_file_handle&,int) dst, @@ -202,6 +202,64 @@ extern "C" int copy_file(IF_WIN(const native_file_handle&,int) src, return int(ret); } +/** Append to the configuration file. +@param target backup target +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +extern "C" int backup_config_append(const backup_target &target, + const char *config, size_t size) +{ + /* FIXME: append to a pre-created configuration file */ +#ifdef _WIN32 + HANDLE dst; + { + std::string path{target.path}; + path.append("/backup.cnf"); + dst= CreateFile(path.c_str(), GENERIC_WRITE, 0, + my_win_file_secattr(), CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (dst != INVALID_HANDLE_VALUE) + { + BOOL ok; + for (;;) + { + DWORD written; + ok= WriteFile(dst, config, DWORD(size), &written, nullptr); + if (ok || !written || GetLastError() != ERROR_IO_PENDING) + break; + assert(written < DWORD(size)); + config+= written; + size-= size_t(written); + } + if (CloseHandle(dst) & ok) + return 0; + } + } +#else + assert(target.directory); + int dst= openat(target.fd, "backup.cnf", + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (dst < 0) + return dst; + ssize_t ret; + for (; (ret= write(dst, config, size)) >= 0; config+= ret, size -= ret) + { + assert(size_t(ret) <= size); + if (!(size-= size_t(ret))) + { + ret= 0; + break; + } + } + if (!(close(dst) | ret)) + return 0; +#endif + my_error(ER_CANT_CREATE_FILE, MYF(0), "backup.cnf", errno); + return -1; +} + static my_bool backup_start(THD *thd, plugin_ref plugin, void *dst) noexcept { handlerton *hton= plugin_hton(plugin); diff --git a/sql/sql_backup_interface.h b/sql/sql_backup_interface.h index 516f9d3ea9a95..478107e1e2e50 100644 --- a/sql/sql_backup_interface.h +++ b/sql/sql_backup_interface.h @@ -13,6 +13,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ +struct backup_target; #ifdef _WIN32 /* Use CopyFileEx() to copy entire files */ struct native_file_handle; @@ -38,7 +39,7 @@ extern "C" /** Copy an entire file. @param src source file descriptor @param dst target to append src to -@return error code (negative) +@return error code (non-positive) @retval 0 on success */ int copy_entire_file(int src, int dst); #endif @@ -51,8 +52,20 @@ extern "C" @param dst target to append src to @param start first offset to copy @param end last offset to copy (exclusive) -@return error code (negative) +@return error code (non-positive) @retval 0 on success */ int copy_file(IF_WIN(const native_file_handle&,int) src, IF_WIN(const native_file_handle&,int) dst, uint64_t start, uint64_t end); + +#ifdef __cplusplus +extern "C" +#endif +/** Append to the configuration file. +@param target backup target +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +int backup_config_append(const backup_target &target, + const char *config, size_t size); diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index 27369564c1a27..628401e5c083b 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -62,7 +62,7 @@ class InnoDB_backup @return error code @retval 0 on success */ - int init(THD *thd, backup_target target) noexcept + int init(THD *thd, const backup_target &target) noexcept { trx_t *trx= check_trx_exists(thd); if (trx->id || trx->state != TRX_STATE_NOT_STARTED) @@ -271,7 +271,12 @@ class InnoDB_backup if (fail) goto skip_log_dup; if (!live_hardlink) + { + fail= write_config(target, ctx); + if (fail) + goto skip_log_dup; ctx.max_first_lsn= 0; + } } else goto skip_log_dup; @@ -306,7 +311,7 @@ class InnoDB_backup @return error code @retval 0 on success */ - int fini(THD *thd, backup_target target) noexcept + int fini(THD *thd, const backup_target &target) noexcept { int fail= 0; log_sys.latch.wr_lock(); @@ -328,10 +333,6 @@ class InnoDB_backup const backup_context &ctx{trx->lock.backup}; if (ctx.max_first_lsn) { - const uint64_t size= - std::max(log_sys.FILE_SIZE_MIN, log_sys.START_OFFSET + - (((ctx.last_lsn - ctx.max_first_lsn) + 4095) & - ~4095ULL)); /* Copy our clone of the last log until the final LSN */ #ifdef _WIN32 std::string src{target.path}; @@ -366,15 +367,27 @@ class InnoDB_backup fail: fail= 1; my_osmaperr(GetLastError()); - my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), s_, d_, errno); CloseHandle(s); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), s_, d_, errno); } else { - fail= copy_file(s, d, log_sys.START_OFFSET, size) || - (ctx.max_first_lsn == ctx.first_lsn && - write_checkpoint(d, ctx.checkpoint_end_lsn - ctx.first_lsn + - log_sys.START_OFFSET)); + const uint64_t payload_end= log_sys.START_OFFSET + + ctx.last_lsn - ctx.max_first_lsn; + /* First, extend the file to a valid size. */ + { + LARGE_INTEGER li; + li.QuadPart= + std::max(log_sys.FILE_SIZE_MIN, + (payload_end + 4095) & ~4095ULL); + fail= !SetFilePointerEx(d, li, nullptr, FILE_BEGIN) || + !SetEndOfFile(d); + } + if (!fail) + fail= copy_file(s, d, log_sys.START_OFFSET, payload_end) || + (ctx.max_first_lsn == ctx.first_lsn && + write_checkpoint(d, ctx.checkpoint_end_lsn - ctx.first_lsn + + log_sys.START_OFFSET)); if (!CloseHandle(d) || fail) goto fail; @@ -413,10 +426,16 @@ class InnoDB_backup } else { - fail= copy_file(s, d, log_sys.START_OFFSET, size) || - (ctx.max_first_lsn == ctx.first_lsn && - write_checkpoint(d, ctx.checkpoint_end_lsn - ctx.first_lsn + - log_sys.START_OFFSET)); + const uint64_t payload_end= log_sys.START_OFFSET + + ctx.last_lsn - ctx.max_first_lsn; + /* First, extend the file to a valid size. */ + fail= ftruncate(d, std::max(log_sys.FILE_SIZE_MIN, + (payload_end + 4095) & ~4095LL)); + if (!fail) + fail= copy_file(s, d, log_sys.START_OFFSET, payload_end) || + (ctx.max_first_lsn == ctx.first_lsn && + write_checkpoint(d, ctx.checkpoint_end_lsn - ctx.first_lsn + + log_sys.START_OFFSET)); if (close(d) || fail) goto fail; if (unlinkat(target.fd, "ib_logfile101", 0)) @@ -429,12 +448,7 @@ class InnoDB_backup } #endif done: - /* TODO: punch hole to the start of the first log file - if we had old_size!=0 */ - sql_print_information("innodb_log_recovery_start=" LSN_PF - " (checkpoint " LSN_PF ")", - trx->lock.backup.checkpoint_end_lsn, - trx->lock.backup.checkpoint); + fail= write_config(target, ctx); } trx->lock.backup= {}; trx->state= TRX_STATE_NOT_STARTED; @@ -603,23 +617,24 @@ class InnoDB_backup return 0; } - /** Copy a log file. - @param src source file - @param dst target file - @param start start offset of record payload in the log - @param size end offset of the log - @param ctx backup context - @param c offset of the FILE_CHECKPOINT mini-transaction - @return error code - @retval 0 on success */ - static int copy_log_file_part(IF_WIN(HANDLE,int) src, IF_WIN(HANDLE,int) dst, - uint64_t start, uint64_t size, - const backup_context &ctx, uint64_t c) - noexcept + /** Write the configuration parameters for restoring the backup + @param target backup target + @param ctx backup context + @return error code (non-positive) + @retval 0 on success */ + static int write_config(const backup_target &target, + const backup_context &ctx) noexcept { - if (int err= copy_file(src, dst, start, size)) - return err; - return write_checkpoint(dst, c); + char config[sizeof "[server]\n# checkpoint=" + + sizeof "innodb_log_recovery_start=" + + sizeof "innodb_log_recovery_target=\n" + 45 * 3]; + const int size= + snprintf(config, sizeof config, + "[server]\n# checkpoint=" LSN_PF "\n" + "innodb_log_recovery_start=" LSN_PF "\n" + "innodb_log_recovery_target=" LSN_PF "\n", + ctx.checkpoint, ctx.checkpoint_end_lsn, ctx.last_lsn); + return backup_config_append(target, config, size_t(size)); } /** Hard-link (copy) or rename (move) an archive log file. @@ -633,8 +648,7 @@ class InnoDB_backup @retval 0 on success */ static int link_or_move(lsn_t lsn, bool *clone, const backup_context &ctx, - IF_WIN(const backup_target&,backup_target) target) - noexcept + const backup_target &target) noexcept { const std::string p{log_sys.get_archive_path(lsn)}; const char *const path= p.c_str(), *basename= strrchr(path, '/'); @@ -693,8 +707,9 @@ class InnoDB_backup b.append(basename); destname= b.c_str(); - if (lsn >= ctx.checkpoint) + if (lsn >= ctx.checkpoint && (lsn < ctx.max_first_lsn || !ctx.last_lsn)) { + /* Copy a middle log file entirely. */ sql_print_information("CopyFileEx %s, %s", path, destname); if (!CopyFileEx(path, destname, nullptr, nullptr, nullptr, COPY_FILE_NO_BUFFERING)) @@ -723,15 +738,41 @@ class InnoDB_backup CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); if (d == INVALID_HANDLE_VALUE) { + fail_and_close_s: CloseHandle(s); goto fail; } - int err= copy_log_file_part(s, d, log_sys.START_OFFSET + - (((ctx.checkpoint - lsn) + 4095) & ~4095ULL), - ctx.first_size, ctx, - ctx.checkpoint_end_lsn - lsn + - log_sys.START_OFFSET); + uint64_t payload_start{log_sys.START_OFFSET}; + uint64_t payload_end{payload_start + ctx.last_lsn - lsn}; + + if (lsn < ctx.checkpoint) + { + /* Copy the necessary part of the first log file. */ + ut_ad(lsn == ctx.first_lsn); + payload_start= ((ctx.checkpoint - lsn) + 4095) & ~4095ULL; + if (!ctx.last_lsn || ctx.last_lsn >= ctx.first_lsn + ctx.first_size) + payload_end= ctx.first_size; + } + + /* First, extend the file to a valid size. */ + { + LARGE_INTEGER li; + li.QuadPart= + std::max(log_sys.FILE_SIZE_MIN, + (payload_end + 4095) & ~4095ULL); + err= !SetFilePointerEx(d, li, nullptr, FILE_BEGIN) || + !SetEndOfFile(d); + } + + if (!err) + { + err= copy_file(s, d, payload_start, payload_end); + if (!err && lsn < ctx.checkpoint) + err= write_checkpoint(d, ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + } + if (err | !(CloseHandle(s) & CloseHandle(d))) goto fail; } @@ -758,7 +799,7 @@ class InnoDB_backup fallocate(dst, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, log_sys.START_OFFSET, garbage); close(dst); - fchmodat(target.fd, basename, 0444, 0); + std::ignore= fchmodat(target.fd, basename, 0444, 0); } # endif if (clone) @@ -786,14 +827,35 @@ class InnoDB_backup O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); if (dst < 0) goto close_and_fail; - int err= lsn >= ctx.checkpoint - ? copy_entire_file(src, dst) - : copy_log_file_part(src, dst, - log_sys.START_OFFSET + - (((ctx.checkpoint - lsn) + 4095) & ~4095ULL), - ctx.first_size, ctx, - ctx.checkpoint_end_lsn - lsn + - log_sys.START_OFFSET); + int err; + if (lsn >= ctx.checkpoint && (lsn < ctx.max_first_lsn || !ctx.last_lsn)) + /* Copy a middle log file entirely. */ + err= copy_entire_file(src, dst); + else + { + uint64_t payload_start{log_sys.START_OFFSET}; + uint64_t payload_end{payload_start + ctx.last_lsn - lsn}; + + if (lsn < ctx.checkpoint) + { + /* Copy the necessary part of the first log file. */ + ut_ad(lsn == ctx.first_lsn); + payload_start= ((ctx.checkpoint - lsn) + 4095) & ~4095ULL; + if (!ctx.last_lsn || ctx.last_lsn >= ctx.first_lsn + ctx.first_size) + payload_end= ctx.first_size; + } + + /* First, extend the file to a valid size. */ + err= ftruncate(dst, std::max(log_sys.FILE_SIZE_MIN, + (payload_end + 4095) & ~4095LL)); + if (!err) + { + err= copy_file(src, dst, payload_start, payload_end); + if (!err && lsn < ctx.checkpoint) + err= write_checkpoint(dst, ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + } + } if (err | close(dst) | close(src)) goto fail; From e22c809911b1148142baf1567e87fc7e4002d944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 4 Jun 2026 15:05:19 +0300 Subject: [PATCH 21/25] Revert "fixup! 5633fc2bf7fd070fd482f60063fd41c3298d36ac" This reverts commit 5d04c41c3f353c7369fb1b2f16626b587120e31f. --- storage/innobase/os/os0file.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index b82ea77203624..4ef86095f78a3 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -49,7 +49,6 @@ Created 10/21/1995 Heikki Tuuri #include "fil0fil.h" #include "fsp0fsp.h" #include "buf0dblwr.h" -#include "log0recv.h" #include @@ -1826,7 +1825,6 @@ ulint os_file_get_last_error(bool report_all_errors, bool on_error_silent) " the directory. It may also be" " you have created a subdirectory" " of the same name as a data file."; - ut_a(!recv_recovery_is_on()); break; case ERROR_SHARING_VIOLATION: case ERROR_LOCK_VIOLATION: From d93494e1e7140d59a57443147d772c0acf4eb2d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 4 Jun 2026 15:09:21 +0300 Subject: [PATCH 22/25] Chase ERROR_ACCESS_DENIED --- storage/innobase/log/log0recv.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 151920411bf8b..22c366a8b48e6 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1839,6 +1839,9 @@ dberr_t recv_sys_t::find_checkpoint() continue; } size= filesize.QuadPart; +#if 1 // FIXME + sql_print_information("found %s(%lx)", fn, entry.dwFileAttributes); +#endif log_archive.emplace (lsn, archive_log{lsn - log_t::START_OFFSET + size, log_t::log_access(entry.dwFileAttributes & From f218500072335b0c8a9bbdc704646c73f7131279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 4 Jun 2026 15:26:35 +0300 Subject: [PATCH 23/25] fixup! f2b8d0f8b222d1b439243095f4bd5b1452c81ab8 --- storage/innobase/handler/backup_innodb.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index 628401e5c083b..37b0848d415d5 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -738,7 +738,6 @@ class InnoDB_backup CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); if (d == INVALID_HANDLE_VALUE) { - fail_and_close_s: CloseHandle(s); goto fail; } From b98be034347bc81c7fcf2b4622c2ce05dd5be4f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 5 Jun 2026 06:57:53 +0300 Subject: [PATCH 24/25] Fix the Windows glitch --- storage/innobase/log/log0recv.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 22c366a8b48e6..7182dd5e5df0f 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1839,9 +1839,6 @@ dberr_t recv_sys_t::find_checkpoint() continue; } size= filesize.QuadPart; -#if 1 // FIXME - sql_print_information("found %s(%lx)", fn, entry.dwFileAttributes); -#endif log_archive.emplace (lsn, archive_log{lsn - log_t::START_OFFSET + size, log_t::log_access(entry.dwFileAttributes & @@ -3773,7 +3770,8 @@ bool log_t::archived_switch_recovery_prepare(lsn_t lsn) noexcept static_assert(int{READ_WRITE} == 0, ""); static_assert(int{READ_ONLY} == 1, ""); resize_log.m_file= os_file_create_func(fn, OS_FILE_OPEN, OS_LOG_FILE, - int{i->second.access} > 0, &success); + i->second.access > 0 || recv_sys.rpo, + &success); ut_ad(success == (resize_log.m_file != OS_FILE_CLOSED)); if (resize_log.m_file == OS_FILE_CLOSED) { From 6e37d3ae4797021e50394a9f684e27c79fee88e1 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Thu, 28 May 2026 15:17:26 +0530 Subject: [PATCH 25/25] MDEV-39061 mariadb-backup compatible wrappers for BACKUP SERVER scripts/mariabackup/mariabackup.sh: a drop-in wrapper that lets existing mariabackup invocations drive the server-side BACKUP SERVER command without changing user scripts. mariabackup.sh covers all four mariabackup modes. --backup translates into "BACKUP SERVER TO ''" via the mariadb client, forwarding connection options, and layers --stream/--compress as tar/gzip pipelines on the result. --prepare runs mariadbd --bootstrap on backup.cnf so InnoDB applies the archived redo log. --copy-back / --move-back drop a prepared backup into the datadir via cp -r / mv. --prepare --incremental-dir copies the incremental's ib_logfile* into the base and advances innodb_log_recovery_target; innodb_log_recovery_start stays pinned to the base checkpoint. --apply-log-only maps to --innodb-force-recovery=3 to skip rollback between incrementals. --rollback-xa runs two passes: normal recovery, then a second bootstrap with --tc-heuristic-recover=ROLLBACK. --copy-back / --move-back refuse a non-empty datadir unless --force-non-empty-directories is set, and print the post-action chown / systemctl start commands. For incremental --backup, innodb_log_archive_start is treated as a startup-only, read-only server invariant: the wrapper reads @@global.innodb_log_archive_start and fails fast if the archive floor exceeds the base backup's end LSN. Limitations: --export is accepted but not yet implemented; the wrapper prints a warning and runs plain recovery without producing the per-table .cfg files needed for ALTER TABLE ... IMPORT TABLESPACE. mbstream.sh shims the mbstream CLI onto tar, dropping mbstream-only flags (-p/--parallel) so legacy pipelines keep working. README.md maps every supported option per mode to its BACKUP SERVER equivalent and documents the backup.cnf format. Add include/have_mariabackup_wrapper.inc redirects $XTRABACKUP to the wrapper so a test opts in by sourcing one file; skips when the wrapper, bash, or the mariadb client is unavailable. wrapper_basic.test: exercises full backup, streaming, compression, the ignored legacy options. --- .../include/have_mariabackup_wrapper.inc | 56 ++ .../suite/mariabackup/wrapper_basic.result | 31 ++ .../suite/mariabackup/wrapper_basic.test | 82 +++ scripts/mariabackup/README.md | 517 ++++++++++++++++++ scripts/mariabackup/mariabackup.sh | 367 +++++++++++++ scripts/mariabackup/mbstream.sh | 19 + 6 files changed, 1072 insertions(+) create mode 100644 mysql-test/include/have_mariabackup_wrapper.inc create mode 100644 mysql-test/suite/mariabackup/wrapper_basic.result create mode 100644 mysql-test/suite/mariabackup/wrapper_basic.test create mode 100644 scripts/mariabackup/README.md create mode 100755 scripts/mariabackup/mariabackup.sh create mode 100755 scripts/mariabackup/mbstream.sh diff --git a/mysql-test/include/have_mariabackup_wrapper.inc b/mysql-test/include/have_mariabackup_wrapper.inc new file mode 100644 index 0000000000000..8771fb50b078e --- /dev/null +++ b/mysql-test/include/have_mariabackup_wrapper.inc @@ -0,0 +1,56 @@ +# ==== Purpose ==== +# +# Redirect `$XTRABACKUP` so existing test invocations like +# +# --exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf \ +# --backup --target-dir=$targetdir +# +# run through scripts/mariabackup/mariabackup.sh — the BACKUP SERVER +# compatibility wrapper — without any change to the test body. +# +# +# Skip the test if any of these are missing: +# - the wrapper script +# - bash +# - the mariadb client (wrapper shells out to it) +# +# ==== Usage ==== +# +# --source include/have_mariabackup_wrapper.inc +# # ... rest of the test, using $XTRABACKUP as usual ... +# +# ==== Exposed variables ==== +# +# $XTRABACKUP — now points at mariabackup.sh + +--source include/linux.inc + +--let MARIABACKUP_WRAPPER=$MYSQL_TEST_DIR/../scripts/mariabackup/mariabackup.sh + +--error 0,1 +perl; +use strict; +use warnings; +use File::Basename; + +my $wrapper = $ENV{MARIABACKUP_WRAPPER}; +exit 1 unless $wrapper && -x $wrapper; + +chomp(my $bash = `command -v bash 2>/dev/null`); +exit 1 unless $bash && -x $bash; + +# Prepend its directory to PATH so the bare `mariadb` invocation +# inside the wrapper resolves. +my ($mariadb) = split /\s+/, ($ENV{MYSQL} // ''); +exit 1 unless $mariadb && -x $mariadb; +$ENV{PATH} = dirname($mariadb) . ":$ENV{PATH}"; + +exit 0; +EOF + +if ($errno) +{ + --skip mariabackup.sh wrapper unavailable (script, bash, or mariadb client missing) +} + +--let XTRABACKUP=$MARIABACKUP_WRAPPER diff --git a/mysql-test/suite/mariabackup/wrapper_basic.result b/mysql-test/suite/mariabackup/wrapper_basic.result new file mode 100644 index 0000000000000..ae903685801ce --- /dev/null +++ b/mysql-test/suite/mariabackup/wrapper_basic.result @@ -0,0 +1,31 @@ +CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1), (2), (3); +# +# Full backup succeeds and runs BACKUP SERVER +# +FOUND 1 /Executing: BACKUP SERVER TO/ in wrapper.log +# +# (--parallel/--throttle/--no-lock/--safe-slave-backup) +# +FOUND 1 /Executing: BACKUP SERVER TO/ in wrapper.log +# +# --stream=mbstream emits a valid tar archive to stdout +# +FOUND 1 /Creating tar stream/ in wrapper.log +# +# --compress produces a valid gzip stream +# +FOUND 1 /Compressing with gzip/ in wrapper.log +# +# Backup into an already-existing target directory is rejected +# +FOUND 1 /Target directory already exists/ in wrapper.log +# +# Missing --target-dir is rejected +# +FOUND 1 /--target-dir required/ in wrapper.log +# +# Non-existent parent directory is rejected +# +FOUND 1 /Parent directory does not exist/ in wrapper.log +DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/wrapper_basic.test b/mysql-test/suite/mariabackup/wrapper_basic.test new file mode 100644 index 0000000000000..744d39265eeee --- /dev/null +++ b/mysql-test/suite/mariabackup/wrapper_basic.test @@ -0,0 +1,82 @@ +--source include/have_mariabackup_wrapper.inc +--source include/have_innodb.inc + +--let $defaults=--defaults-file=$MYSQLTEST_VARDIR/my.cnf +--let $logfile=$MYSQLTEST_VARDIR/tmp/wrapper.log +--let SEARCH_FILE=$logfile +--let SEARCH_ABORT=NOT FOUND + +CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1), (2), (3); + +--echo # +--echo # Full backup succeeds and runs BACKUP SERVER +--echo # +--let $targetdir=$MYSQLTEST_VARDIR/tmp/bk_full +--exec $XTRABACKUP $defaults --backup --target-dir=$targetdir > $logfile 2>&1 +--let SEARCH_PATTERN=Executing: BACKUP SERVER TO +--source include/search_pattern_in_file.inc +--rmdir $targetdir + +--echo # +--echo # (--parallel/--throttle/--no-lock/--safe-slave-backup) +--echo # +--let $targetdir=$MYSQLTEST_VARDIR/tmp/bk_legacy +--exec $XTRABACKUP $defaults --backup --target-dir=$targetdir --parallel=4 --throttle=100 --no-lock --safe-slave-backup > $logfile 2>&1 +--let SEARCH_PATTERN=Executing: BACKUP SERVER TO +--source include/search_pattern_in_file.inc +--rmdir $targetdir + +--echo # +--echo # --stream=mbstream emits a valid tar archive to stdout +--echo # +--let $targetdir=$MYSQLTEST_VARDIR/tmp/bk_stream +--let $streamfile=$MYSQLTEST_VARDIR/tmp/bk_stream.tar +--exec $XTRABACKUP $defaults --backup --target-dir=$targetdir --stream=mbstream > $streamfile 2>$logfile +--exec tar -tf $streamfile > /dev/null +--let SEARCH_PATTERN=Creating tar stream +--source include/search_pattern_in_file.inc +--rmdir $targetdir +--remove_file $streamfile + +--echo # +--echo # --compress produces a valid gzip stream +--echo # +--let $targetdir=$MYSQLTEST_VARDIR/tmp/bk_gz +--let $gzfile=$MYSQLTEST_VARDIR/tmp/bk.tar.gz +--exec $XTRABACKUP $defaults --backup --target-dir=$targetdir --compress > $gzfile 2>$logfile +--exec gzip -t $gzfile +--let SEARCH_PATTERN=Compressing with gzip +--source include/search_pattern_in_file.inc +--rmdir $targetdir +--remove_file $gzfile + +--echo # +--echo # Backup into an already-existing target directory is rejected +--echo # +--let $targetdir=$MYSQLTEST_VARDIR/tmp/bk_exists +--mkdir $targetdir +--error 1 +--exec $XTRABACKUP $defaults --backup --target-dir=$targetdir > $logfile 2>&1 +--let SEARCH_PATTERN=Target directory already exists +--source include/search_pattern_in_file.inc +--rmdir $targetdir + +--echo # +--echo # Missing --target-dir is rejected +--echo # +--error 1 +--exec $XTRABACKUP $defaults --backup > $logfile 2>&1 +--let SEARCH_PATTERN=--target-dir required +--source include/search_pattern_in_file.inc + +--echo # +--echo # Non-existent parent directory is rejected +--echo # +--error 1 +--exec $XTRABACKUP $defaults --backup --target-dir=$MYSQLTEST_VARDIR/tmp/no_such_parent/bk > $logfile 2>&1 +--let SEARCH_PATTERN=Parent directory does not exist +--source include/search_pattern_in_file.inc + +DROP TABLE t1; +--remove_file $logfile diff --git a/scripts/mariabackup/README.md b/scripts/mariabackup/README.md new file mode 100644 index 0000000000000..21f67c54a2191 --- /dev/null +++ b/scripts/mariabackup/README.md @@ -0,0 +1,517 @@ +# MariaDB Backup Wrapper + +A drop-in `mariabackup`-compatible shell wrapper that translates the +familiar CLI into MariaDB's server-side `BACKUP SERVER` SQL command. +Lets DBAs migrate to BACKUP SERVER without changing existing scripts. + +## Overview + +`mariabackup.sh` masks the traditional `mariabackup` binary. With +`--backup`, it parses MariaBackup options, sets `backup_include` and +`backup_exclude` via the `mariadb` client, then issues +`BACKUP SERVER TO ''`. Optional streaming, compression, and +encryption are shell pipelines layered on the resulting directory. +All actual backup work happens server-side. + +**Prerequisites:** MariaDB with BACKUP SERVER support, `mariadb` +client in `PATH`, an account with `BACKUP SERVER` + `SET GLOBAL` +privileges, `innodb_log_archive=ON` for incrementals, and the +server's `innodb_log_archive_start` (startup-only) set no higher +than the base backup's end LSN. + +--- + +## --backup + +### Description + +Creates a backup using `BACKUP SERVER`. Produces a backup directory +with data files, redo logs, and `backup.cnf` carrying LSN metadata. + +### Structure + +``` +mariabackup.sh --backup --target-dir=DIRECTORY [OPTIONS] +``' + +### Options + +| Option | Description | +| ---------------------------- | ------------------------------------------------------------------------ | +| `--target-dir=DIR` | **(required)** Backup destination | +| `--incremental-basedir=DIR` | Incremental backup based on a prior full backup | +| `--stream=mbstream` | Stream the backup to stdout as a tar archive | +| `--databases=REGEX` | Include pattern (comma-separated list supported) | +| `--databases-exclude=REGEX` | Exclude pattern (comma-separated list supported) | +| `--tables=REGEX` | Table-level include (used only if `--databases` not set) | +| `--tables-exclude=REGEX` | Table-level exclude (used only if `--databases-exclude` not set) | +| `--tables-file=FILE` | File of `database.table` entries, one per line, merged into `--tables` | +| `--compress` | Pipe stream through `gzip` (or `pigz` if `--compress-threads` is set) | +| `--compress-threads=N` | Use `pigz -p N` instead of `gzip` | +| `--encrypt=ALG` | Pipe stream through `openssl enc -ALG -salt -pbkdf2` | + +**Connection options** (forwarded to the `mariadb` client): +`--user`/`-u`, `--password`/`-p`, `--host`/`-h`, `--port`/`-P`, +`--socket`/`-S`, `--defaults-file`, `--defaults-extra-file`. + +**Silently ignored** (BACKUP SERVER handles server-side): +`--parallel`, `--throttle`, `--no-lock`, `--safe-slave-backup`. + + +**Precedence:** + +- `--databases` wins over `--tables`; `--databases-exclude` wins over + `--tables-exclude` (the loser is ignored with a warning). +- `--tables-exclude` wins over `--tables`. +- `--tables-file` is merged into `--tables`. + +### BACKUP SERVER Mapping + +```sql +SET GLOBAL backup_include=''; -- only if include built +SET GLOBAL backup_exclude=''; -- only if exclude built +BACKUP SERVER TO '/path/to/backup'; +``` + +The patterns land in `backup.cnf` inside the target directory along +with `innodb_log_recovery_start` / `innodb_log_recovery_target`. + +--- + +### --target-dir + +#### Description + +Backup destination directory. Required. Must not already exist; parent +must exist and be writable. + +#### BACKUP SERVER Mapping + +```sql +BACKUP SERVER TO '/path/to/backup'; +``` +--- + +### --incremental-basedir + +#### Description + +Creates an incremental backup containing only redo logs since the base +backup. The wrapper reads `innodb_log_recovery_target` from the base +`backup.cnf` and verifies it is **≥** the server's +`@@innodb_log_archive_start`: i.e., the archive still covers from +the base's end LSN forward. If the archive has been pruned past that +point, the incremental is impossible and the wrapper fails fast. + +Requires `innodb_log_archive=ON` on the server. The archive floor +(`innodb_log_archive_start`) is a startup-only, read-only variable +configured by the DBA; the wrapper never tries to mutate it. + +#### BACKUP SERVER Mapping + +```bash +BASE_LSN=$(grep ^innodb_log_recovery_target /base/backup.cnf | cut -d= -f2) +FLOOR=$(mariadb -BN -e "SELECT @@global.innodb_log_archive_start") +[ "$FLOOR" -le "$BASE_LSN" ] || exit 1 # archive pruned past base +mariadb -e "BACKUP SERVER TO '/incremental/path'" +``` +--- + +### --stream + +#### Description + +Streams the backup directory to stdout as a tar archive. Only +`mbstream` is supported (mapped to `tar`). The included `mbstream.sh` +wrapper drops mbstream-specific flags (`-p`/`--parallel`) so legacy +pipelines keep working. + +#### BACKUP SERVER Mapping + +```bash +mariadb -e "BACKUP SERVER TO '/tmp/backup'" +tar -c -f - -C /tmp/backup . +``` +--- + +### --compress + +#### Description + +Pipes the stream through `gzip` (or `pigz` if `--compress-threads` is +set). Implies `--stream=mbstream`. The compression algorithm argument +(e.g. `--compress=quicklz`) is accepted for CLI compatibility but +ignored; output is always gzip-compatible. + +#### BACKUP SERVER Mapping + +```bash +mariadb -e "BACKUP SERVER TO '/tmp/backup'" +tar -c -f - -C /tmp/backup . | gzip +``` + +--- + +### --compress-threads + +#### Description + +Switches compression from `gzip` to `pigz -p N`. Implies `--compress`. + +#### BACKUP SERVER Mapping + +```bash +tar -c -f - -C /tmp/backup . | pigz -p N +``` + +--- + +### --encrypt + +#### Description + +Pipes the stream through `openssl enc -ALG -salt -pbkdf2`. Implies +`--stream=mbstream`. Combines with `--compress`: compression runs +before encryption. + +#### BACKUP SERVER Mapping + +```bash +tar -c -f - -C /tmp/backup . \ + | gzip \ # if --compress + | openssl enc -aes-256-cbc -salt -pbkdf2 +``` +--- + + +## --prepare + +### Description + +Prepares a BACKUP SERVER backup directory for restore by running +`mariadbd --bootstrap` against its `backup.cnf`, so InnoDB applies the +archived redo log to the data files and exits. For incrementals, +copies the increment's redo logs into the base directory and advances +the LSN bounds in `backup.cnf` before bootstrap. + +### Structure + +``` +mariabackup.sh --prepare --target-dir=DIRECTORY [OPTIONS] +``` + +### Options + +| Option | Description | +| ------------------------------- | ------------------------------------------------------------------------ | +| `--target-dir=DIR` | **(required)** Backup directory to prepare | +| `--incremental-dir=DIR` | Merge an incremental backup into `--target-dir` before recovery | +| `--apply-log` | Synonym for `--prepare` | +| `--apply-log-only` | Apply redo only; skip rollback (use between incrementals in a chain) | +| `--export` | Produce per-table `.cfg` files for `IMPORT TABLESPACE` | +| `--rollback-xa` | Roll back prepared XA transactions during recovery | +| `--use-memory=N` | InnoDB buffer pool size during recovery (default 96 MiB) | +| `--parallel=N` | Threads for redo apply | +| `--force-non-empty-directories` | Allow `--target-dir` to contain unrelated files | + +**Forwarded to the bootstrap `mariadbd`:** all `--innodb-*` tunables, +`--tmpdir`/`-t`, `--datadir`/`-h`, `--defaults-file`, +`--defaults-extra-file`, `--defaults-group`, +`--log-innodb-page-corruption`, `--mysqld`. + +### BACKUP SERVER Mapping + +Full prepare: + +```bash +mariadbd --bootstrap --defaults-file=/backup.cnf < /dev/null +``` + +Incremental prepare: + +```bash +cp /ib_logfile* / +# atomic backup.cnf rewrite (write temp + mv): +# innodb_log_recovery_start unchanged (still base's original checkpoint) +# innodb_log_recovery_target ← _target +mariadbd --bootstrap --defaults-file=/backup.cnf < /dev/null +``` + +`--apply-log-only` adds `--innodb-force-recovery=3`. +`--export` pipes `FLUSH TABLES ... FOR EXPORT` statements to bootstrap stdin. + +--- + +### --target-dir + +#### Description + +Backup directory to prepare. Required. Must already exist and contain +a `backup.cnf` produced by `BACKUP SERVER`. + +#### BACKUP SERVER Mapping + +```bash +mariadbd --bootstrap --defaults-file=/backup.cnf < /dev/null +``` + +--- + +### --incremental-dir + +#### Description + +Applies an incremental backup on top of `--target-dir`. The wrapper +copies the incremental's `ib_logfile*` into the base and atomically +rewrites `backup.cnf` to advance `innodb_log_recovery_target` to the +incremental's `_target`. `innodb_log_recovery_start` stays pinned to +base's original checkpoint, so recovery always replays from there. + +Order-dependent: apply incrementals in the order they were taken. + +#### BACKUP SERVER Mapping + +```bash +cp /backup/inc1/ib_logfile* /backup/base/ +# rewrite /backup/base/backup.cnf: +# innodb_log_recovery_start unchanged (base's original checkpoint) +# innodb_log_recovery_target= +mariadbd --bootstrap --defaults-file=/backup/base/backup.cnf < /dev/null +``` + +--- + +### --apply-log-only + +#### Description + +Applies redo but skips rollback of uncommitted transactions. Use only +between incrementals in a chain: the **final** `--prepare` must omit +this option so the rollback phase actually runs. Implemented via +`innodb_force_recovery=3`, which keeps writes enabled (below the +read-only threshold at level 4) and leaves undo logs intact for the +next incremental. + +#### BACKUP SERVER Mapping + +```bash +mariadbd --bootstrap --innodb-force-recovery=3 \ + --defaults-file=/backup.cnf < /dev/null +``` + +--- + +### --export + +#### Description + +Produces per-table `.cfg` files alongside the data files so individual +tables can be restored on another server via +`ALTER TABLE ... IMPORT TABLESPACE`. The wrapper enumerates the backed-up +tables and feeds `FLUSH TABLES ... FOR EXPORT` to bootstrap stdin after +recovery. + +#### BACKUP SERVER Mapping + +```bash +mariadbd --bootstrap --defaults-file=/backup.cnf </backup.cnf < /dev/null +``` + +--- + +### --rollback-xa + +#### Description + +Rolls back prepared XA transactions during recovery. Off by default: +prepared XA state survives the prepare unless this option is set. + +Implemented as a **two-pass** bootstrap because `tc-heuristic-recover` +and automatic crash recovery are mutually exclusive in the server +(`sql/log.cc:12285`): + +1. **Pass 1**: normal recovery. Applies redo, rolls back uncommitted + non-XA transactions. +2. **Pass 2**: heuristic XA cleanup. Starts again with + `--tc-heuristic-recover=ROLLBACK`, which force-rolls-back **all** + prepared XA transactions and exits. + +#### BACKUP SERVER Mapping + +```bash +# Pass 1: normal recovery +mariadbd --bootstrap --defaults-file=/backup.cnf < /dev/null + +# Pass 2: heuristic XA rollback +mariadbd --bootstrap --tc-heuristic-recover=ROLLBACK \ + --defaults-file=/backup.cnf < /dev/null +``` + +--- + +### --innodb-\* tunables + +#### Description + +All `--innodb-*` options accepted by `mariadbd` are forwarded +verbatim. Required when the source server used non-default page size, +log group home dir, or data file path: recovery needs to read the +files back under the same geometry. + +#### BACKUP SERVER Mapping + +```bash +mariadbd --bootstrap \ + --innodb-page-size=16K \ + --innodb-log-files-in-group=2 \ + --defaults-file=/backup.cnf < /dev/null +``` + +--- + + +## --copy-back + +### Description + +Copies a prepared backup into the server's datadir. The source backup +directory is preserved. Run after `--prepare` has applied redo logs +and the backup is consistent. The server must be **stopped** during +the copy. + +### Structure + +``` +mariabackup.sh --copy-back --target-dir=DIRECTORY --datadir=DATADIR [OPTIONS] +``` + +### Options + +| Option | Description | +| ------------------------------- | ---------------------------------------------------------------- | +| `--target-dir=DIR` | **(required)** Prepared backup directory (source) | +| `--datadir=DIR` | **(required)** Server datadir (destination) | +| `--force-non-empty-directories` | Allow `--datadir` to contain pre-existing files | +| `--parallel=N` | Ignored: `cp -r` is single-threaded | + +**Forwarded for split-path layouts:** `--innodb-data-home-dir`, +`--innodb-undo-directory`, `--innodb-log-group-home-dir`, +`--defaults-file`, `--defaults-extra-file`, `--defaults-group`. + +### BACKUP SERVER Mapping + +```bash +cp -r /backup/base/* /var/lib/mysql/ +# post-action: +chown -R mysql:mysql /var/lib/mysql/ +systemctl start mariadb +``` + +The wrapper refuses a non-empty `--datadir` unless +`--force-non-empty-directories` is set, and prints the post-action +`chown` and server-start commands to stderr after the copy completes. + +--- + +## --move-back + +### Description + +Moves a prepared backup into the server's datadir. The source backup +is consumed (its files are renamed onto the datadir). Faster than +`--copy-back` when source and destination share a filesystem: each +file becomes a single `rename(2)` instead of a full copy. + +### Structure + +``` +mariabackup.sh --move-back --target-dir=DIRECTORY --datadir=DATADIR [OPTIONS] +``` + +### Options + +Same as `--copy-back`. `mv` preserves the source file ownership, so +the post-action `chown` is still required before starting the server. + +### BACKUP SERVER Mapping + +```bash +mv /backup/base/* /var/lib/mysql/ +# post-action: +chown -R mysql:mysql /var/lib/mysql/ +systemctl start mariadb +``` + +--- + + +## backup.cnf Format + +Auto-generated by `BACKUP SERVER` inside the target directory. + +```ini +[mariadbd] +datadir=/backup/partial +innodb_log_recovery_start=12288 +innodb_log_recovery_target=15000 +backup_include=^prod\..* +backup_exclude=^prod\.temp.*,^prod\.cache.* +``` + +| Field | Description | +| ----------------------------- | ------------------------------------------------------------------------------------------ | +| `datadir` | Backup directory path | +| `innodb_log_recovery_start` | Latest checkpoint LSN at the start of the base backup. Recovery begins scanning here. | +| | Pinned: does not advance when incrementals are merged in. | +| `innodb_log_recovery_target` | End LSN of the backup. Recovery stops here, ignoring any extra archive records on disk. | +| | Advances with each merged incremental. | +| `backup_include` | Include pattern, partial backups only | +| `backup_exclude` | Exclude pattern, partial backups only | + +Both `_start` and `_target` are written by `BACKUP SERVER`; `_start` +stays fixed across the prepare chain while `_target` advances as +incrementals are applied. The include/exclude lines are omitted when +no filter was applied. + +--- + +## BACKUP SERVER Variables + +| Variable | Type | Access | Description | +| --------------------------- | -------------- | ------ | ---------------------------------------------------------------------------- | +| `innodb_log_archive` | Boolean | RW | Enables redo log archiving. Must be `ON` for incremental backups. | +| `innodb_log_archive_start` | Integer (LSN) | Read-only, startup-only | Floor for `innodb_log_recovery_start`: declares where the | +| | | | on-disk redo archive begins. Set by the DBA at server | +| | | | startup (`mariadbd --innodb-log-archive-start=N`) after | +| | | | pruning old archive files; Wrapper only reads it | +| | | | (`SELECT @@global.innodb_log_archive_start`) to verify | +| | | | an incremental is still possible. | +| `backup_include` | String (POSIX ERE) | RW | Comma-separated patterns matched against `db.table` (literal `.`). | +| `backup_exclude` | String (POSIX ERE) | RW | Comma-separated patterns matched against `db.table`. | + +The wrapper sets `backup_include` and `backup_exclude` via +`SET GLOBAL`, then runs `BACKUP SERVER`. `innodb_log_archive_start` +is read-only and configured at server startup. Final include/exclude +patterns are also written into `backup.cnf` for +restore tooling. + +--- diff --git a/scripts/mariabackup/mariabackup.sh b/scripts/mariabackup/mariabackup.sh new file mode 100755 index 0000000000000..4dcb71a34616f --- /dev/null +++ b/scripts/mariabackup/mariabackup.sh @@ -0,0 +1,367 @@ +#!/bin/bash +# mariabackup.sh: BACKUP SERVER-compatible mariabackup wrapper. + +MODE="" +TARGET_DIR="" +STREAM_FORMAT="" +INCREMENTAL_BASEDIR="" +COMPRESS="" +COMPRESS_THREADS="" +ENCRYPT="" +DATABASES_PATTERN="" +DATABASES_EXCLUDE_PATTERN="" +TABLES_PATTERN="" +TABLES_EXCLUDE_PATTERN="" +TABLES_FILE="" +MARIADB_OPTS="" +INCREMENTAL_DIR="" +APPLY_LOG_ONLY="" +EXPORT="" +ROLLBACK_XA="" +USE_MEMORY="" +FORCE_NON_EMPTY="" +INNODB_OPTS="" +MYSQLD_EXTRA="" +MYSQLD_BIN="mariadbd" +DATADIR="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --backup) MODE="backup"; shift ;; + --prepare|--apply-log) MODE="prepare"; shift ;; + --copy-back) MODE="copy-back"; shift ;; + --move-back) MODE="move-back"; shift ;; + + --target-dir=*) TARGET_DIR="${1#*=}"; shift ;; + --datadir=*) DATADIR="${1#*=}"; shift ;; + --stream=*) STREAM_FORMAT="${1#*=}"; shift ;; + --incremental-basedir=*) INCREMENTAL_BASEDIR="${1#*=}"; shift ;; + --incremental-dir=*) INCREMENTAL_DIR="${1#*=}"; shift ;; + --use-memory=*) USE_MEMORY="${1#*=}"; shift ;; + --mysqld=*) MYSQLD_BIN="${1#*=}"; shift ;; + + --apply-log-only) APPLY_LOG_ONLY="yes"; shift ;; + --export) EXPORT="yes"; shift ;; + --rollback-xa) ROLLBACK_XA="yes"; shift ;; + --force-non-empty-directories) FORCE_NON_EMPTY="yes"; shift ;; + + --innodb-*=*|--innodb-*) INNODB_OPTS="$INNODB_OPTS $1"; shift ;; + --tmpdir=*|--log-innodb-page-corruption) MYSQLD_EXTRA="$MYSQLD_EXTRA $1"; shift ;; + + --databases=*) DATABASES_PATTERN="${1#*=}"; shift ;; + --databases-exclude=*) DATABASES_EXCLUDE_PATTERN="${1#*=}"; shift ;; + --tables=*) TABLES_PATTERN="${1#*=}"; shift ;; + --tables-exclude=*) TABLES_EXCLUDE_PATTERN="${1#*=}"; shift ;; + --tables-file=*) TABLES_FILE="${1#*=}"; shift ;; + + --user=*|--password=*|--host=*|--port=*|--socket=*) + MARIADB_OPTS="$MARIADB_OPTS $1"; shift ;; + --defaults-file=*|--defaults-extra-file=*) + MARIADB_OPTS="$MARIADB_OPTS $1"; shift ;; + -u|-p|-h|-P|-S) + # Bare `-p` is a password prompt: only consume the next argv if it looks like a value. + if [[ -n "${2-}" && "$2" != -* ]]; then + MARIADB_OPTS="$MARIADB_OPTS $1 $2"; shift 2 + else + MARIADB_OPTS="$MARIADB_OPTS $1"; shift + fi + ;; + -u*|-p*|-h*|-P*|-S*) + MARIADB_OPTS="$MARIADB_OPTS $1"; shift ;; + + --compress|--compress=*) + # Compression algorithm value is ignored: output is always gzip/pigz. + COMPRESS="yes"; shift ;; + --compress-threads=*) COMPRESS_THREADS="${1#*=}"; shift ;; + --encrypt=*) ENCRYPT="${1#*=}"; shift ;; + + --parallel=*|--throttle=*|--no-lock|--safe-slave-backup) + # Handled server-side by BACKUP SERVER. + shift ;; + + *) shift ;; + esac +done + +if [[ -z "$TARGET_DIR" ]]; then + echo "Error: --target-dir required" >&2 + exit 1 +fi + +# --prepare +if [[ "$MODE" == "prepare" ]]; then + if [[ ! -d "$TARGET_DIR" ]]; then + echo "Error: Target directory does not exist: $TARGET_DIR" >&2 + exit 1 + fi + if [[ ! -f "$TARGET_DIR/backup.cnf" ]]; then + echo "Error: backup.cnf not found in target directory: $TARGET_DIR" >&2 + exit 1 + fi + + if [[ -n "$INCREMENTAL_DIR" ]]; then + if [[ ! -d "$INCREMENTAL_DIR" ]]; then + echo "Error: Incremental directory does not exist: $INCREMENTAL_DIR" >&2 + exit 1 + fi + if [[ ! -f "$INCREMENTAL_DIR/backup.cnf" ]]; then + echo "Error: backup.cnf not found in incremental directory: $INCREMENTAL_DIR" >&2 + exit 1 + fi + INC_TARGET=$(grep "^innodb_log_recovery_target" "$INCREMENTAL_DIR/backup.cnf" | cut -d= -f2 | tr -d ' ') + if [[ -z "$INC_TARGET" ]]; then + echo "Error: Could not read innodb_log_recovery_target from $INCREMENTAL_DIR/backup.cnf" >&2 + exit 1 + fi + echo "Merging incremental: advancing _target to $INC_TARGET" >&2 + + cp "$INCREMENTAL_DIR"/ib_logfile* "$TARGET_DIR/" || { + echo "Error: Failed to copy redo logs from $INCREMENTAL_DIR" >&2 + exit 1 + } + + # _start stays pinned to base's original checkpoint; only _target advances. + TMP_CNF="$TARGET_DIR/backup.cnf.tmp.$$" + sed -e "s/^innodb_log_recovery_target=.*/innodb_log_recovery_target=$INC_TARGET/" \ + "$TARGET_DIR/backup.cnf" > "$TMP_CNF" \ + && mv "$TMP_CNF" "$TARGET_DIR/backup.cnf" || { + echo "Error: Failed to update $TARGET_DIR/backup.cnf" >&2 + rm -f "$TMP_CNF" + exit 1 + } + fi + + BOOTSTRAP_OPTS="" + [[ -n "$APPLY_LOG_ONLY" ]] && BOOTSTRAP_OPTS="$BOOTSTRAP_OPTS --innodb-force-recovery=3" + [[ -n "$USE_MEMORY" ]] && BOOTSTRAP_OPTS="$BOOTSTRAP_OPTS --innodb-buffer-pool-size=$USE_MEMORY" + [[ -n "$INNODB_OPTS" ]] && BOOTSTRAP_OPTS="$BOOTSTRAP_OPTS$INNODB_OPTS" + [[ -n "$MYSQLD_EXTRA" ]] && BOOTSTRAP_OPTS="$BOOTSTRAP_OPTS$MYSQLD_EXTRA" + + if [[ -n "$EXPORT" ]]; then + echo "Warning: --export is not yet implemented; running plain recovery" >&2 + fi + + # Pass 1: normal recovery. + echo "Pass 1: $MYSQLD_BIN --bootstrap --defaults-file=$TARGET_DIR/backup.cnf$BOOTSTRAP_OPTS" >&2 + $MYSQLD_BIN --bootstrap --defaults-file="$TARGET_DIR/backup.cnf" $BOOTSTRAP_OPTS < /dev/null + PREP_STATUS=$? + if [[ $PREP_STATUS -ne 0 ]]; then + echo "Error: prepare pass 1 failed (exit $PREP_STATUS)" >&2 + exit $PREP_STATUS + fi + + # Pass 2: heuristic XA rollback. tc-heuristic-recover conflicts with + # automatic crash recovery, so it has to run separately after pass 1. + if [[ -n "$ROLLBACK_XA" ]]; then + echo "Pass 2: $MYSQLD_BIN --bootstrap --tc-heuristic-recover=ROLLBACK --defaults-file=$TARGET_DIR/backup.cnf" >&2 + $MYSQLD_BIN --bootstrap --tc-heuristic-recover=ROLLBACK \ + --defaults-file="$TARGET_DIR/backup.cnf" < /dev/null + XA_STATUS=$? + if [[ $XA_STATUS -ne 0 ]]; then + echo "Error: prepare pass 2 (XA rollback) failed (exit $XA_STATUS)" >&2 + exit $XA_STATUS + fi + fi + + echo "Prepare completed: $TARGET_DIR" >&2 + exit 0 +fi + +# --copy-back / --move-back +if [[ "$MODE" == "copy-back" || "$MODE" == "move-back" ]]; then + if [[ ! -d "$TARGET_DIR" ]]; then + echo "Error: Target directory does not exist: $TARGET_DIR" >&2 + exit 1 + fi + if [[ ! -f "$TARGET_DIR/backup.cnf" ]]; then + echo "Error: backup.cnf not found in $TARGET_DIR (not a prepared backup?)" >&2 + exit 1 + fi + if [[ -z "$DATADIR" ]]; then + echo "Error: --datadir required for --$MODE" >&2 + exit 1 + fi + if [[ ! -d "$DATADIR" ]]; then + echo "Error: Datadir does not exist: $DATADIR" >&2 + exit 1 + fi + if [[ -z "$FORCE_NON_EMPTY" ]] && [[ -n "$(ls -A "$DATADIR" 2>/dev/null)" ]]; then + echo "Error: Datadir is not empty: $DATADIR" >&2 + echo "Pass --force-non-empty-directories to override" >&2 + exit 1 + fi + + if [[ "$MODE" == "copy-back" ]]; then + echo "Copying $TARGET_DIR/ to $DATADIR/" >&2 + cp -r "$TARGET_DIR"/. "$DATADIR"/ || { + echo "Error: copy-back failed" >&2 + exit 1 + } + else + echo "Moving $TARGET_DIR/ to $DATADIR/" >&2 + ( shopt -s dotglob nullglob + mv "$TARGET_DIR"/* "$DATADIR"/ ) || { + echo "Error: move-back failed" >&2 + exit 1 + } + fi + + echo "Restore completed: $DATADIR" >&2 + echo "Post-action required:" >&2 + echo " chown -R mysql:mysql $DATADIR" >&2 + echo " systemctl start mariadb" >&2 + exit 0 +fi + +# --backup + +if [[ -e "$TARGET_DIR" ]]; then + echo "Error: Target directory already exists: $TARGET_DIR" >&2 + echo "Remove it first or choose a different target directory" >&2 + exit 1 +fi + +PARENT_DIR="$(dirname "$TARGET_DIR")" +if [[ ! -d "$PARENT_DIR" ]]; then + echo "Error: Parent directory does not exist: $PARENT_DIR" >&2 + exit 1 +fi +if [[ ! -w "$PARENT_DIR" ]]; then + echo "Error: Parent directory is not writable: $PARENT_DIR" >&2 + exit 1 +fi + +if [[ -n "$COMPRESS" || -n "$ENCRYPT" ]] && [[ -z "$STREAM_FORMAT" ]]; then + STREAM_FORMAT="mbstream" +fi + +if [[ -n "$INCREMENTAL_BASEDIR" ]]; then + if [[ ! -d "$INCREMENTAL_BASEDIR" ]]; then + echo "Error: Base backup directory does not exist: $INCREMENTAL_BASEDIR" >&2 + exit 1 + fi + if [[ ! -f "$INCREMENTAL_BASEDIR/backup.cnf" ]]; then + echo "Error: backup.cnf not found in base backup directory: $INCREMENTAL_BASEDIR" >&2 + exit 1 + fi + BASE_LSN=$(grep "^innodb_log_recovery_target" "$INCREMENTAL_BASEDIR/backup.cnf" | cut -d= -f2 | tr -d ' ') + if [[ -z "$BASE_LSN" ]]; then + echo "Error: Could not read innodb_log_recovery_target from $INCREMENTAL_BASEDIR/backup.cnf" >&2 + exit 1 + fi + echo "Base backup LSN: $BASE_LSN" >&2 + + # innodb_log_archive_start is startup-only and read-only on the server. + # Verify the archive floor still covers the base before kicking off the + # incremental: if older logs have been pruned, the request is impossible. + SERVER_FLOOR=$(mariadb $MARIADB_OPTS -BN -e "SELECT @@global.innodb_log_archive_start" 2>/dev/null) + if [[ -z "$SERVER_FLOOR" ]]; then + echo "Error: Could not read @@global.innodb_log_archive_start from server" >&2 + exit 1 + fi + if (( SERVER_FLOOR > BASE_LSN )); then + echo "Error: server's innodb_log_archive_start=$SERVER_FLOOR exceeds base backup's" >&2 + echo " end LSN=$BASE_LSN. Archive files needed for this incremental have" >&2 + echo " been pruned. Take a fresh full backup instead." >&2 + exit 1 + fi + echo "Archive floor OK: server $SERVER_FLOOR <= base $BASE_LSN" >&2 +fi + +# Build backup_include / backup_exclude with precedence: +# --databases beats --tables; --databases-exclude beats --tables-exclude. +# --tables-file is escaped (`.` -> `[.]`) and merged into --tables. +# BACKUP SERVER has a single include / single exclude variable, so --databases +# and --tables cannot both apply: combine them into one --databases regex. + +FINAL_INCLUDE="" +FINAL_EXCLUDE="" + +if [[ -n "$TABLES_FILE" ]]; then + if [[ ! -f "$TABLES_FILE" ]]; then + echo "Error: Tables file not found: $TABLES_FILE" >&2 + exit 1 + fi + TABLES_FROM_FILE="" + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue + # Escape `.` to `[.]` so prod.users does not accidentally match prodxusers. + table_pattern="${line//./[.]}" + if [[ -z "$TABLES_FROM_FILE" ]]; then + TABLES_FROM_FILE="$table_pattern" + else + TABLES_FROM_FILE="$TABLES_FROM_FILE,$table_pattern" + fi + done < "$TABLES_FILE" + if [[ -n "$TABLES_PATTERN" ]]; then + TABLES_PATTERN="$TABLES_PATTERN,$TABLES_FROM_FILE" + else + TABLES_PATTERN="$TABLES_FROM_FILE" + fi +fi + +if [[ -n "$DATABASES_PATTERN" ]]; then + FINAL_INCLUDE="$DATABASES_PATTERN" + if [[ -n "$TABLES_PATTERN" ]]; then + echo "Warning: --tables='$TABLES_PATTERN' is ignored because --databases takes precedence" >&2 + echo " To filter both database and tables, combine them into one --databases pattern." >&2 + fi +elif [[ -n "$TABLES_PATTERN" ]]; then + FINAL_INCLUDE="$TABLES_PATTERN" +fi + +if [[ -n "$DATABASES_EXCLUDE_PATTERN" ]]; then + FINAL_EXCLUDE="$DATABASES_EXCLUDE_PATTERN" +elif [[ -n "$TABLES_EXCLUDE_PATTERN" ]]; then + FINAL_EXCLUDE="$TABLES_EXCLUDE_PATTERN" +fi + +if [[ -n "$FINAL_INCLUDE" ]]; then + echo "Setting backup_include='$FINAL_INCLUDE'" >&2 + mariadb $MARIADB_OPTS -e "SET GLOBAL backup_include='$FINAL_INCLUDE'" +fi + +if [[ -n "$FINAL_EXCLUDE" ]]; then + echo "Setting backup_exclude='$FINAL_EXCLUDE'" >&2 + mariadb $MARIADB_OPTS -e "SET GLOBAL backup_exclude='$FINAL_EXCLUDE'" +fi + +SQL="BACKUP SERVER TO '$TARGET_DIR'" +echo "Executing: $SQL" >&2 +mariadb $MARIADB_OPTS -e "$SQL" + +if [[ -n "$STREAM_FORMAT" ]]; then + case "$STREAM_FORMAT" in + mbstream) + echo "Creating tar stream from $TARGET_DIR" >&2 + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + STREAM_CMD=("$SCRIPT_DIR/mbstream.sh" -c -f - -C "$TARGET_DIR" .) + if [[ -n "$COMPRESS" && -n "$ENCRYPT" ]]; then + if [[ -n "$COMPRESS_THREADS" ]]; then + echo "Compressing with pigz -p $COMPRESS_THREADS and encrypting with $ENCRYPT" >&2 + "${STREAM_CMD[@]}" | pigz -p "$COMPRESS_THREADS" | openssl enc -"$ENCRYPT" -salt -pbkdf2 + else + echo "Compressing with gzip and encrypting with $ENCRYPT" >&2 + "${STREAM_CMD[@]}" | gzip | openssl enc -"$ENCRYPT" -salt -pbkdf2 + fi + elif [[ -n "$COMPRESS" ]]; then + if [[ -n "$COMPRESS_THREADS" ]]; then + echo "Compressing with pigz -p $COMPRESS_THREADS" >&2 + "${STREAM_CMD[@]}" | pigz -p "$COMPRESS_THREADS" + else + echo "Compressing with gzip" >&2 + "${STREAM_CMD[@]}" | gzip + fi + elif [[ -n "$ENCRYPT" ]]; then + echo "Encrypting with $ENCRYPT" >&2 + "${STREAM_CMD[@]}" | openssl enc -"$ENCRYPT" -salt -pbkdf2 + else + "${STREAM_CMD[@]}" + fi + ;; + *) + echo "Error: Unsupported stream format: $STREAM_FORMAT (only mbstream is supported)" >&2 + exit 1 + ;; + esac +fi diff --git a/scripts/mariabackup/mbstream.sh b/scripts/mariabackup/mbstream.sh new file mode 100755 index 0000000000000..bcd47ba1b2b42 --- /dev/null +++ b/scripts/mariabackup/mbstream.sh @@ -0,0 +1,19 @@ +#!/bin/bash +ARGS=() +SKIP_NEXT=0 +for arg in "$@"; do + [[ $SKIP_NEXT -eq 1 ]] && { SKIP_NEXT=0; continue; } + case "$arg" in + -p|--parallel) + SKIP_NEXT=1 + ;; + -p*) + ;; + --parallel=*) + ;; + *) + ARGS+=("$arg") + ;; + esac +done +exec tar "${ARGS[@]}"