From 4769a43b51c58b0fd02b1164a6f80ab916d21903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 25 Jun 2026 16:18:23 +0300 Subject: [PATCH 01/14] MDEV-14992 BACKUP SERVER This introduces a basic driver Sql_cmd_backup, storage engine interfaces, and basic copying of InnoDB, Aria, and MyISAM files. For streaming, we aim to generate streams that are compatible with GNU tar --format=oldgnu, which is also supported by the built-in BSD tar on FreeBSD, macOS as well as Microsoft Windows. That is, to extract files from a backup stream, you can use the standard tar utility of the operating system, instead of anything nonstandard like xbstream or mbstream. TODO: Support partial backup and restore. This should be done by implementing some configuration parameters as well as a predicate that checks if a file name pattern should be included. TODO: Back up ENGINE=RocksDB. TODO: Refactor the crude first implementation of Aria_backup that is based on the work of Andrzej Jarzabek, to copy as many files as concurrently as possible while holding the minimum amount of locks. --- backup_target: A structured data type to represent a target directory. On Microsoft Windows, we must use directory paths because there is no variant of CopyFileEx() that would work on file handles. backup_sink: Wraps a per-thread output stream as well as storage engine specific context. handlerton::backup_start(), handlerton::backup_end(): Invoked at the start or end of a backup phase, in the thread that executes a BACKUP SERVER statement. handlerton::backup_step(): A backup step that can be invoked from multiple threads concurrently, between the execution of the corresponding handlerton::backup_start() and handlerton::backup_end() of the same phase. copy_entire_file(): A file copying service for POSIX systems. copy_file(): A sparse file-copying service for all systems. backup_stream_append_async(): A variant of backup_stream_append() where the source file region is guaranteed to be immutable after the call returns. We must not use Linux sendfile(2) for copying data files that may be modified in place, because it could introduce a race condition between a page write that runs concurrently with a child process that is reading the data from the pipe. InnoDB_backup::context: Backup context, attached to backup_sink so that context can continue to exist between the time a BACKUP SERVER releases all locks and another BACKUP SERVER starts executing, with innodb_backup pointing to the new backup, while the old backup is still being finished. fil_space_t::write_or_backup: Keep track of in-flight page writes and pending backup operation. We must not allow them concurrently, because that could lead into torn pages in the backup. fil_space_t::backup_end: The first page number that is not being backed up (by default 0, to indicate that no backup is in progress). fil_space_t::BACKUP_BATCH_SIZE: The number of preceding pages that will be covered by fil_space_t::backup_end. This is the unit of "page range locking" during InnoDB backup. log_sys.backup: Whether BACKUP SERVER is in progress. The purpose of this is to make BACKUP SERVER prevent the concurrent execution of SET GLOBAL innodb_log_archive=OFF or SET GLOBAL innodb_log_file_size when innodb_log_archive=OFF. log_sys.archived_checkpoint: Keep track of the earliest available checkpoint, corresponding to log_sys.archived_lsn. This reflects SET GLOBAL innodb_log_recovery_start (which is settable now), for incremental backup. buf_flush_list_space(): Check for concurrent backup before writing each page. This is inefficient, but this function may be invoked from multiple threads concurrently, and it cannot be changed easily, especially for fil_crypt_thread(). --- libmysqld/CMakeLists.txt | 1 + mysql-test/collections/buildbot_suites.bat | 1 + mysql-test/main/backup_server.result | 6 + mysql-test/main/backup_server.test | 10 + mysql-test/main/backup_server_locking.result | 17 + mysql-test/main/backup_server_locking.test | 31 + mysql-test/main/grant_backup_server.result | 27 + mysql-test/main/grant_backup_server.test | 29 + mysql-test/main/mysqld--help.result | 2 +- mysql-test/mariadb-test-run.pl | 1 + .../suite/backup/backup_innodb,debug.rdiff | 16 + .../suite/backup/backup_innodb.combinations | 4 + mysql-test/suite/backup/backup_innodb.result | 60 + mysql-test/suite/backup/backup_innodb.test | 104 ++ .../suite/backup/backup_stream,debug.rdiff | 16 + .../suite/backup/backup_stream.combinations | 4 + mysql-test/suite/backup/backup_stream.result | 35 + mysql-test/suite/backup/backup_stream.test | 89 ++ mysql-test/suite/backup/suite.pm | 19 + .../perfschema/r/max_program_zero.result | 2 +- .../suite/perfschema/r/ortho_iter.result | 2 +- .../perfschema/r/privilege_table_io.result | 2 +- .../r/start_server_disable_idle.result | 2 +- .../r/start_server_disable_stages.result | 2 +- .../r/start_server_disable_statements.result | 2 +- .../start_server_disable_transactions.result | 2 +- .../r/start_server_disable_waits.result | 2 +- .../perfschema/r/start_server_innodb.result | 2 +- .../r/start_server_low_index.result | 2 +- .../r/start_server_low_table_lock.result | 2 +- .../r/start_server_no_account.result | 2 +- .../r/start_server_no_cond_class.result | 2 +- .../r/start_server_no_cond_inst.result | 2 +- .../r/start_server_no_file_class.result | 2 +- .../r/start_server_no_file_inst.result | 2 +- .../perfschema/r/start_server_no_host.result | 2 +- .../perfschema/r/start_server_no_index.result | 2 +- .../perfschema/r/start_server_no_mdl.result | 2 +- .../r/start_server_no_memory_class.result | 2 +- .../r/start_server_no_mutex_class.result | 2 +- .../r/start_server_no_mutex_inst.result | 2 +- ..._server_no_prepared_stmts_instances.result | 2 +- .../r/start_server_no_rwlock_class.result | 2 +- .../r/start_server_no_rwlock_inst.result | 2 +- .../r/start_server_no_setup_actors.result | 2 +- .../r/start_server_no_setup_objects.result | 2 +- .../r/start_server_no_socket_class.result | 2 +- .../r/start_server_no_socket_inst.result | 2 +- .../r/start_server_no_stage_class.result | 2 +- .../r/start_server_no_stages_history.result | 2 +- ...start_server_no_stages_history_long.result | 2 +- .../start_server_no_statements_history.result | 2 +- ...t_server_no_statements_history_long.result | 2 +- .../r/start_server_no_table_hdl.result | 2 +- .../r/start_server_no_table_inst.result | 2 +- .../r/start_server_no_table_lock.result | 2 +- .../r/start_server_no_thread_class.result | 2 +- .../r/start_server_no_thread_inst.result | 2 +- ...tart_server_no_transactions_history.result | 2 +- ...server_no_transactions_history_long.result | 2 +- .../perfschema/r/start_server_no_user.result | 2 +- .../r/start_server_no_waits_history.result | 2 +- .../start_server_no_waits_history_long.result | 2 +- .../perfschema/r/start_server_off.result | 2 +- .../suite/perfschema/r/start_server_on.result | 2 +- .../r/start_server_variables.result | 2 +- .../r/statement_program_lost_inst.result | 2 +- .../suite/sys_vars/r/sysvars_innodb.result | 2 +- sql/CMakeLists.txt | 1 + sql/handler.h | 92 ++ sql/mysqld.cc | 1 + sql/sql_backup.cc | 838 +++++++++++++ sql/sql_backup.h | 53 + sql/sql_backup_interface.h | 178 +++ sql/sql_command.h | 1 + sql/sql_parse.cc | 4 +- sql/sql_yacc.yy | 28 +- sql/sys_vars.inl | 2 + storage/innobase/CMakeLists.txt | 2 + storage/innobase/buf/buf0flu.cc | 99 +- storage/innobase/handler/backup_innodb.cc | 1110 +++++++++++++++++ storage/innobase/handler/backup_innodb.h | 58 + storage/innobase/handler/ha_innodb.cc | 54 +- storage/innobase/include/fil0fil.h | 47 + storage/innobase/include/log0log.h | 62 +- storage/innobase/log/log0log.cc | 78 +- storage/innobase/log/log0recv.cc | 4 +- storage/innobase/os/os0file.cc | 10 +- storage/maria/CMakeLists.txt | 1 + storage/maria/ha_maria.cc | 4 + storage/maria/ma_backup.cc | 414 ++++++ storage/maria/ma_backup.h | 58 + 92 files changed, 3645 insertions(+), 124 deletions(-) create mode 100644 mysql-test/main/backup_server.result create mode 100644 mysql-test/main/backup_server.test create mode 100644 mysql-test/main/backup_server_locking.result create mode 100644 mysql-test/main/backup_server_locking.test create mode 100644 mysql-test/main/grant_backup_server.result create mode 100644 mysql-test/main/grant_backup_server.test create mode 100644 mysql-test/suite/backup/backup_innodb,debug.rdiff create mode 100644 mysql-test/suite/backup/backup_innodb.combinations create mode 100644 mysql-test/suite/backup/backup_innodb.result create mode 100644 mysql-test/suite/backup/backup_innodb.test create mode 100644 mysql-test/suite/backup/backup_stream,debug.rdiff create mode 100644 mysql-test/suite/backup/backup_stream.combinations create mode 100644 mysql-test/suite/backup/backup_stream.result create mode 100644 mysql-test/suite/backup/backup_stream.test create mode 100644 mysql-test/suite/backup/suite.pm create mode 100644 sql/sql_backup.cc create mode 100644 sql/sql_backup.h create mode 100644 sql/sql_backup_interface.h create mode 100644 storage/innobase/handler/backup_innodb.cc create mode 100644 storage/innobase/handler/backup_innodb.h create mode 100644 storage/maria/ma_backup.cc create mode 100644 storage/maria/ma_backup.h diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index 80b202aa4c03e..59f386a86edae 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -166,6 +166,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/opt_hints.cc ../sql/opt_hints.h ../sql/opt_trace_ddl_info.cc ../sql/opt_trace_ddl_info.h ../sql/sql_path.cc + ../sql/sql_backup.cc ${GEN_SOURCES} ${MYSYS_LIBWRAP_SOURCE} ) diff --git a/mysql-test/collections/buildbot_suites.bat b/mysql-test/collections/buildbot_suites.bat index 053057872bf40..129a2f67d79de 100644 --- a/mysql-test/collections/buildbot_suites.bat +++ b/mysql-test/collections/buildbot_suites.bat @@ -6,6 +6,7 @@ innodb,^ versioning,^ plugins,^ mariabackup,^ +backup,^ roles,^ auth_gssapi,^ mysql_sha2,^ diff --git a/mysql-test/main/backup_server.result b/mysql-test/main/backup_server.result new file mode 100644 index 0000000000000..7a064d7e5fe11 --- /dev/null +++ b/mysql-test/main/backup_server.result @@ -0,0 +1,6 @@ +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; diff --git a/mysql-test/main/backup_server.test b/mysql-test/main/backup_server.test new file mode 100644 index 0000000000000..50cf326d39838 --- /dev/null +++ b/mysql-test/main/backup_server.test @@ -0,0 +1,10 @@ +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory diff --git a/mysql-test/main/backup_server_locking.result b/mysql-test/main/backup_server_locking.result new file mode 100644 index 0000000000000..6e49a09e3e873 --- /dev/null +++ b/mysql-test/main/backup_server_locking.result @@ -0,0 +1,17 @@ +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +BACKUP STAGE START; +connect backup,localhost,root; +SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR 70100: Query was interrupted: execution time limit 0.1 sec exceeded +connection default; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't execute the command as you have a BACKUP STAGE active +BACKUP STAGE END; +connection backup; +SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +disconnect backup; +connection default; diff --git a/mysql-test/main/backup_server_locking.test b/mysql-test/main/backup_server_locking.test new file mode 100644 index 0000000000000..4e769b61b2f20 --- /dev/null +++ b/mysql-test/main/backup_server_locking.test @@ -0,0 +1,31 @@ +--source include/not_embedded.inc +--source include/count_sessions.inc + +--mkdir $MYSQLTEST_VARDIR/some_directory +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +BACKUP STAGE START; +--connect (backup,localhost,root) +--error ER_STATEMENT_TIMEOUT +evalp SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +--connection default + +--error ER_BACKUP_LOCK_IS_ACTIVE +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +BACKUP STAGE END; +--connection backup +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--disconnect backup +--connection default + +--rmdir $MYSQLTEST_VARDIR/some_directory + +--source include/wait_until_count_sessions.inc diff --git a/mysql-test/main/grant_backup_server.result b/mysql-test/main/grant_backup_server.result new file mode 100644 index 0000000000000..a4d85292dee45 --- /dev/null +++ b/mysql-test/main/grant_backup_server.result @@ -0,0 +1,27 @@ +CREATE USER user1@localhost IDENTIFIED BY ''; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON test.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT RELOAD ON test.* TO user1@localhost; +ERROR HY000: Incorrect usage of DB GRANT and GLOBAL PRIVILEGES +GRANT RELOAD ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the SELECT privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +disconnect con1; +connection default; +DROP USER user1@localhost; diff --git a/mysql-test/main/grant_backup_server.test b/mysql-test/main/grant_backup_server.test new file mode 100644 index 0000000000000..726abb19908bb --- /dev/null +++ b/mysql-test/main/grant_backup_server.test @@ -0,0 +1,29 @@ +--source include/not_embedded.inc +CREATE USER user1@localhost IDENTIFIED BY ''; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON test.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +--error ER_WRONG_USAGE +GRANT RELOAD ON test.* TO user1@localhost; +GRANT RELOAD ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +--disconnect con1 +--connection default +DROP USER user1@localhost; diff --git a/mysql-test/main/mysqld--help.result b/mysql-test/main/mysqld--help.result index 623311c170616..e55576862a7c2 100644 --- a/mysql-test/main/mysqld--help.result +++ b/mysql-test/main/mysqld--help.result @@ -2040,7 +2040,7 @@ performance-schema-max-socket-classes 10 performance-schema-max-socket-instances -1 performance-schema-max-sql-text-length 1024 performance-schema-max-stage-classes 170 -performance-schema-max-statement-classes 227 +performance-schema-max-statement-classes 228 performance-schema-max-statement-stack 10 performance-schema-max-table-handles -1 performance-schema-max-table-instances -1 diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index 788927069015e..936e3c4d8fdf6 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -180,6 +180,7 @@ END main- archive- atomic- + backup- binlog- binlog_encryption- binlog_in_engine- diff --git a/mysql-test/suite/backup/backup_innodb,debug.rdiff b/mysql-test/suite/backup/backup_innodb,debug.rdiff new file mode 100644 index 0000000000000..592e1bf5e57ad --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb,debug.rdiff @@ -0,0 +1,16 @@ +--- backup_innodb.result ++++ backup_innodb,debug.result +@@ -20,7 +20,13 @@ + BEGIN; + DELETE FROM t; + connect backup,localhost,root; ++SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; + BACKUP SERVER TO 'target_directory' 4 CONCURRENT; ++connection default; ++SET DEBUG_SYNC='now WAIT_FOR start'; ++INSERT INTO t(a) SELECT * FROM seq_1_to_30000; ++SET DEBUG_SYNC='now SIGNAL resume'; ++connection backup; + disconnect backup; + connection default; + ROLLBACK; diff --git a/mysql-test/suite/backup/backup_innodb.combinations b/mysql-test/suite/backup/backup_innodb.combinations new file mode 100644 index 0000000000000..7fd419bba7692 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.combinations @@ -0,0 +1,4 @@ +[archived] +innodb_log_archive=ON +[circular] +innodb_log_archive=OFF diff --git a/mysql-test/suite/backup/backup_innodb.result b/mysql-test/suite/backup/backup_innodb.result new file mode 100644 index 0000000000000..bff88f510bfe9 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.result @@ -0,0 +1,60 @@ +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (2), (3), (5), (7); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (1), (1), (2), (3), (5); +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; +BACKUP SERVER TO '$target_directory'; +ROLLBACK; +SELECT * FROM t; +a b +1 +BACKUP SERVER TO '$target_directory' WITH '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH '/bin/false'' at line 1 +BACKUP SERVER TO '$target_directory' WITH 4 CONCURRENT '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH 4 CONCURRENT '/bin/false'' at line 1 +BACKUP SERVER WITH '/bin/false'; +ERROR HY000: IO Write error: (...) BACKUP SERVER +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +BACKUP SERVER TO 'target_directory' 4 CONCURRENT; +disconnect backup; +connection default; +ROLLBACK; +SELECT * FROM t; +a b +1 +DELETE FROM t; +DROP TABLE at0, at1; +# restart: --defaults-file=MYSQLTEST_VARDIR/some_directory/backup.cnf --datadir=MYSQLTEST_VARDIR/some_directory +SELECT * FROM t; +a b +1 +DELETE FROM t; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +i +1 +1 +2 +3 +5 +SELECT * FROM at1; +i +2 +3 +5 +7 +DROP TABLE t, at0, at1; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +ERROR 42S02: Table 'test.at0' doesn't exist +SELECT * FROM at1; +ERROR 42S02: Table 'test.at1' doesn't exist +# restart +SELECT * FROM t; +a b +DROP TABLE t; diff --git a/mysql-test/suite/backup/backup_innodb.test b/mysql-test/suite/backup/backup_innodb.test new file mode 100644 index 0000000000000..c58cff8eec21a --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.test @@ -0,0 +1,104 @@ +--source include/have_sequence.inc +--source include/have_innodb.inc +--source include/maybe_debug.inc + +--disable_query_log +call mtr.add_suppression("mariadbd.*: IO Write error:"); +--enable_query_log + +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (2), (3), (5), (7); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (1), (1), (2), (3), (5); + +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; + +--let $target_directory=/tmp/some_directory$MTR_COMBINATION_ARCHIVED +# comment out the following line (and replace all "rmdir" with "exec rm -fr") +# to test cross-filesystem copy +--let $target_directory=$MYSQLTEST_VARDIR/some_directory + +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory + +evalp BACKUP SERVER TO '$target_directory'; +--rmdir $target_directory +ROLLBACK; +# BACKUP SERVER will implicitly commit the current transaction +SELECT * FROM t; + +--error ER_PARSE_ERROR +evalp BACKUP SERVER TO '$target_directory' WITH '/bin/false'; +--error ER_PARSE_ERROR +evalp BACKUP SERVER TO '$target_directory' WITH 4 CONCURRENT '/bin/false'; +--replace_regex /\(.*\)/(...)/ +--error ER_IO_WRITE_ERROR +BACKUP SERVER WITH '/bin/false'; + +BEGIN; +DELETE FROM t; + +--connect backup,localhost,root +if ($have_debug) { +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; +--replace_result $target_directory target_directory +send_eval BACKUP SERVER TO '$target_directory' 4 CONCURRENT; +--connection default +SET DEBUG_SYNC='now WAIT_FOR start'; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +SET DEBUG_SYNC='now SIGNAL resume'; +--connection backup +--reap +} +if (!$have_debug) { +--replace_result $target_directory target_directory +eval BACKUP SERVER TO '$target_directory' 4 CONCURRENT; +} + +--disconnect backup +--connection default +ROLLBACK; +SELECT * FROM t; +DELETE FROM t; +DROP TABLE at0, at1; + +if ($MARIADB_UPGRADE_EXE) { +let target_directory=$target_directory; +perl; +open IN, "<", "$ENV{MYSQLTEST_VARDIR}/my.cnf"; +open OUT, ">>", "$ENV{target_directory}/backup.cnf"; +print OUT while (); +close(IN); +close(OUT); +EOF +} +if (!$MARIADB_UPGRADE_EXE) { + --exec cat $MYSQLTEST_VARDIR/my.cnf >> $target_directory/backup.cnf +} +--let $restart_parameters=--defaults-file=$target_directory/backup.cnf --datadir=$target_directory +--source include/restart_mysqld.inc + +SELECT * FROM t; +# A nonzero innodb_log_recovery_target makes InnoDB read-only. +--error ER_OPEN_AS_READONLY +DELETE FROM t; +# Non-InnoDB tables are read-write. +SELECT * FROM at0; +SELECT * FROM at1; +--error ER_OPEN_AS_READONLY +DROP TABLE t, at0, at1; +--error ER_NO_SUCH_TABLE +SELECT * FROM at0; +--error ER_NO_SUCH_TABLE +SELECT * FROM at1; + +--let $restart_parameters= +--source include/restart_mysqld.inc +SELECT * FROM t; +DROP TABLE t; + +--rmdir $target_directory diff --git a/mysql-test/suite/backup/backup_stream,debug.rdiff b/mysql-test/suite/backup/backup_stream,debug.rdiff new file mode 100644 index 0000000000000..8130727323dec --- /dev/null +++ b/mysql-test/suite/backup/backup_stream,debug.rdiff @@ -0,0 +1,16 @@ +--- backup_stream.result ++++ backup_stream,debug.result +@@ -8,7 +8,13 @@ + BEGIN; + DELETE FROM t; + connect backup,localhost,root; ++SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; + BACKUP SERVER WITH 2 CONCURRENT 'stream.bat'; ++connection default; ++SET DEBUG_SYNC='now WAIT_FOR start'; ++INSERT INTO t(a) SELECT * FROM seq_1_to_30000; ++SET DEBUG_SYNC='now SIGNAL resume'; ++connection backup; + disconnect backup; + connection default; + ROLLBACK; diff --git a/mysql-test/suite/backup/backup_stream.combinations b/mysql-test/suite/backup/backup_stream.combinations new file mode 100644 index 0000000000000..7fd419bba7692 --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.combinations @@ -0,0 +1,4 @@ +[archived] +innodb_log_archive=ON +[circular] +innodb_log_archive=OFF diff --git a/mysql-test/suite/backup/backup_stream.result b/mysql-test/suite/backup/backup_stream.result new file mode 100644 index 0000000000000..e12a2f683c286 --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.result @@ -0,0 +1,35 @@ +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +INSERT INTO t SET a=1; +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (1); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (0); +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +BACKUP SERVER WITH 2 CONCURRENT 'stream.bat'; +disconnect backup; +connection default; +ROLLBACK; +SELECT * FROM t; +a b +1 +DELETE FROM t; +DROP TABLE at0, at1; +# restart: --defaults-file=MYSQLTEST_VARDIR/backup_stream/backup.cnf --datadir=MYSQLTEST_VARDIR/backup_stream +SELECT * FROM at0; +i +0 +SELECT * FROM at1; +i +1 +SELECT * FROM t; +a b +1 +DELETE FROM t; +ERROR HY000: Table 't' is read only +# restart +SELECT * FROM t; +a b +DROP TABLE t; diff --git a/mysql-test/suite/backup/backup_stream.test b/mysql-test/suite/backup/backup_stream.test new file mode 100644 index 0000000000000..649a5557b5b55 --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.test @@ -0,0 +1,89 @@ +--source include/have_sequence.inc +--source include/have_innodb.inc +--source include/maybe_debug.inc + +--disable_query_log +call mtr.add_suppression("mariadbd.*: IO Write error:"); +--enable_query_log + +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +INSERT INTO t SET a=1; + +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (1); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (0); + +--remove_files_wildcard $MYSQL_TMP_DIR/. stream.bat +--write_file $MYSQL_TMP_DIR/stream.bat +#!/bin/sh +exec cat > $MYSQL_TMP_DIR/$1.tar +EOF + +--let $script=$MYSQL_TMP_DIR/stream.bat +if (!$MARIADB_UPGRADE_EXE) { +# Because we may run on Linux /dev/shm which may be mounted as noexec, +# we cannot rely on chmod +x, but must explicitly invoke a shell on the script. +--let $script=/bin/sh $script +} +if ($MARIADB_UPGRADE_EXE) +{ +--exec echo "@cat > %MYSQL_TMP_DIR%\%1.tar" > $MYSQL_TMP_DIR/stream.bat +} + +BEGIN; +DELETE FROM t; + +--connect backup,localhost,root +if ($have_debug) { +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; +--replace_result $script stream.bat +send_eval BACKUP SERVER WITH 2 CONCURRENT '$script'; +--connection default +SET DEBUG_SYNC='now WAIT_FOR start'; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +SET DEBUG_SYNC='now SIGNAL resume'; +--connection backup +--reap +} +if (!$have_debug) { +--replace_result $script stream.bat +eval BACKUP SERVER WITH 2 CONCURRENT '$script'; +} + +--remove_file $MYSQL_TMP_DIR/stream.bat +--disconnect backup +--connection default +ROLLBACK; +SELECT * FROM t; +DELETE FROM t; +DROP TABLE at0, at1; + +--let $target_directory=$MYSQLTEST_VARDIR/backup_stream +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory +--mkdir $target_directory +--exec tar xf $MYSQL_TMP_DIR/1.tar -C $target_directory +--exec tar xf $MYSQL_TMP_DIR/2.tar -C $target_directory + +--exec cat $MYSQLTEST_VARDIR/my.cnf >> $target_directory/backup.cnf + +--let $restart_parameters=--defaults-file=$target_directory/backup.cnf --datadir=$target_directory +--source include/restart_mysqld.inc +SELECT * FROM at0; +SELECT * FROM at1; +SELECT * FROM t; +--error ER_OPEN_AS_READONLY +DELETE FROM t; + +--remove_file $MYSQL_TMP_DIR/1.tar +--remove_file $MYSQL_TMP_DIR/2.tar + +--let $restart_parameters= +--source include/restart_mysqld.inc +SELECT * FROM t; +DROP TABLE t; + +--rmdir $target_directory diff --git a/mysql-test/suite/backup/suite.pm b/mysql-test/suite/backup/suite.pm new file mode 100644 index 0000000000000..7d36a22655efa --- /dev/null +++ b/mysql-test/suite/backup/suite.pm @@ -0,0 +1,19 @@ +package My::Suite::Backup; + +@ISA = qw(My::Suite); +use My::Find; +use File::Basename; +use strict; + +return "Not run for embedded server" if $::opt_embedded_server; + +my $have_cat = index(`echo meowl|cat 2>&1`,"meowl") >= 0; +my $have_tar = `tar --version 2>&1` =~ /tar .*\d\.\d/; + +sub skip_combinations { + my %skip; + $skip{'backup_stream.test'} = 'needs cat,tar' unless $have_cat && $have_tar; + %skip; +} + +bless { }; diff --git a/mysql-test/suite/perfschema/r/max_program_zero.result b/mysql-test/suite/perfschema/r/max_program_zero.result index 047643e06988d..a0e486b2af9c0 100644 --- a/mysql-test/suite/perfschema/r/max_program_zero.result +++ b/mysql-test/suite/perfschema/r/max_program_zero.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 1 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/ortho_iter.result b/mysql-test/suite/perfschema/r/ortho_iter.result index 56c22c8453d8e..589704f4056de 100644 --- a/mysql-test/suite/perfschema/r/ortho_iter.result +++ b/mysql-test/suite/perfschema/r/ortho_iter.result @@ -251,7 +251,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/privilege_table_io.result b/mysql-test/suite/perfschema/r/privilege_table_io.result index 0c82be9f05810..b518052613308 100644 --- a/mysql-test/suite/perfschema/r/privilege_table_io.result +++ b/mysql-test/suite/perfschema/r/privilege_table_io.result @@ -57,7 +57,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_idle.result b/mysql-test/suite/perfschema/r/start_server_disable_idle.result index d0665e3bf4c65..12b956d90769c 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_idle.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_idle.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_stages.result b/mysql-test/suite/perfschema/r/start_server_disable_stages.result index 2ef68328144ff..30ce9d12e56a0 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_stages.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_stages.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_statements.result b/mysql-test/suite/perfschema/r/start_server_disable_statements.result index 0ece2a0c52ed1..4bacdbdfedf68 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_statements.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_statements.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result index ededc09aac95d..3a6831eb2c7ff 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_waits.result b/mysql-test/suite/perfschema/r/start_server_disable_waits.result index 23db9362161e4..a864576dfa979 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_waits.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_waits.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_innodb.result b/mysql-test/suite/perfschema/r/start_server_innodb.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_innodb.result +++ b/mysql-test/suite/perfschema/r/start_server_innodb.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_index.result b/mysql-test/suite/perfschema/r/start_server_low_index.result index 11cade0a2132f..dbd36d2eaa92d 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_index.result +++ b/mysql-test/suite/perfschema/r/start_server_low_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result index 484550095202e..fab64f49d45e6 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_account.result b/mysql-test/suite/perfschema/r/start_server_no_account.result index aab8d3eba9caa..e2cccdba19b43 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_account.result +++ b/mysql-test/suite/perfschema/r/start_server_no_account.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result index 4dfdab9de9f30..44b114013ace2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result index a8d0cbeca3855..27ecb59a40f17 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_class.result b/mysql-test/suite/perfschema/r/start_server_no_file_class.result index fcc01880a7107..5189d36618b05 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result index c56201e7d0a80..533c02383c7b8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_host.result b/mysql-test/suite/perfschema/r/start_server_no_host.result index 662beb3b88a49..e328ea3d3c26a 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_host.result +++ b/mysql-test/suite/perfschema/r/start_server_no_host.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_index.result b/mysql-test/suite/perfschema/r/start_server_no_index.result index ccff0cb113faa..686f430cdc440 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_index.result +++ b/mysql-test/suite/perfschema/r/start_server_no_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mdl.result b/mysql-test/suite/perfschema/r/start_server_no_mdl.result index ebe64409deb0c..c2b6dc3d9eb74 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result index 01a217c534bfd..2a4cdcf8ddd37 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result index 1b3efda5210a9..9b77c7b897c47 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result index 599498915f3f1..3a62653efc18f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result index 73ac1acb9f145..7dd5842809135 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result +++ b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result index 04aa037b960e3..3598c722b9453 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result index e8156711eb3f2..7a60805d2d03f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result index 2d17bd6f49203..76da2883ba6b2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result index 16afee28ee70b..58ac763394a96 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result index 9de0006aa7abc..5a86d51a06231 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 0 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result index bcef0e5c01b05..75ab84f4b3cb8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 0 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result index 1dda39dc79e92..bb750ebd34a26 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 0 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result index 95584521eceb2..6cf3f740fb1be 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result index da6c54b6bbafa..1f2716410a9bc 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result index 09a9a544f5b13..f1078542f69ef 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result index 5ce9874799e8b..a6fc523df2c60 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result index a170fe097fd23..304732cb37927 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 0 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result index 3f009de021d1a..3ef43495d0f22 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 0 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result index db4fe3413106b..20cf1a531ca51 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result index b1b6e614c43ac..e8eb4589fce56 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result index e540f3ce78cbd..4dbf292fc3c10 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result index 80da238ba0ed7..09168c68f9fe8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result index f5d32f0100968..9ec63e991deda 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_user.result b/mysql-test/suite/perfschema/r/start_server_no_user.result index cb249b4e242d3..861b7d897c9c5 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_user.result +++ b/mysql-test/suite/perfschema/r/start_server_no_user.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result index c419aad2995f3..710cef232da58 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result index 0b2bf39ff874f..df4351a8f358f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_off.result b/mysql-test/suite/perfschema/r/start_server_off.result index 17db0395d894b..f3bf7bf7fb15e 100644 --- a/mysql-test/suite/perfschema/r/start_server_off.result +++ b/mysql-test/suite/perfschema/r/start_server_off.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_on.result b/mysql-test/suite/perfschema/r/start_server_on.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_on.result +++ b/mysql-test/suite/perfschema/r/start_server_on.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_variables.result b/mysql-test/suite/perfschema/r/start_server_variables.result index f25f9ab69d1c3..b86116e099cdd 100644 --- a/mysql-test/suite/perfschema/r/start_server_variables.result +++ b/mysql-test/suite/perfschema/r/start_server_variables.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result index 442e212557b55..3e8469b881801 100644 --- a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result +++ b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 2 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 97fd9a7c0ef61..f0834ea3a1e12 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1064,7 +1064,7 @@ NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 18446744073709551615 NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL -READ_ONLY YES +READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_LOG_RECOVERY_TARGET SESSION_VALUE NULL diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index e54e894e1d0fc..fd16d901d64fb 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -163,6 +163,7 @@ SET (SQL_SOURCE grant.cc sql_explain.cc sql_analyze_stmt.cc + sql_backup.cc sql_join_cache.cc create_options.cc multi_range_read.cc opt_histogram_json.cc diff --git a/sql/handler.h b/sql/handler.h index 3ab9e0bcd1a8e..49457902e527d 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -1496,6 +1496,59 @@ struct transaction_participant ulonglong (*prepare_commit_versioned)(THD *thd, ulonglong *trx_id); }; +/** BACKUP SERVER target */ +struct backup_target +{ +#ifdef _WIN32 + /** Target directory path name, or nullptr if streaming */ + const char *path; +#else + /** Target directory descriptor, or -1 if streaming */ + int fd; +#endif +}; + +/** BACKUP SERVER worker specific context */ +struct backup_sink +{ +#ifdef _WIN32 + /** A value indicating an invalid stream */ + static constexpr HANDLE NO_STREAM{INVALID_HANDLE_VALUE}; + /** Target pipe, or NO_STREAM if path!=nullptr */ + HANDLE stream; +#else + /** A value indicating an invalid file descriptor or stream */ + static constexpr int NO_STREAM{-1}; + /** Target pipe, or NO_STREAM if copying to a directory */ + int stream; +#endif + /** storage engine context returned by handlerton::backup_start() */ + void *ha_data; +}; + +/** BACKUP SERVER execution phase */ +enum backup_phase +{ + /** finish backup, possibly after BACKUP_PHASE_ABORT */ + BACKUP_PHASE_FINISH= -2, + /** abort any operation */ + BACKUP_PHASE_ABORT= -1, + /** initial phase; @see MDL_BACKUP_START */ + BACKUP_PHASE_START= 0, + /** copy while new writes to non-transactional tables are blocked; + @see MDL_BACKUP_FLUSH */ + BACKUP_PHASE_NO_BEGIN_NON_TRANS, + /** copy while any writes to non-transactional tables are blocked; + @see MDL_BACKUP_WAIT_FLUSH */ + BACKUP_PHASE_NO_DML_NON_TRANS, + /** copy files while DDL is blocked; @see MDL_BACKUP_WAIT_DDL */ + BACKUP_PHASE_NO_DDL, + /** determine the logical time of the backup and copy any + remaining files while MDL_BACKUP_WAIT_COMMIT is active; + this is followed by BACKUP_PHASE_FINISH */ + BACKUP_PHASE_NO_COMMIT +}; + /* handlerton is a singleton structure - one instance per storage engine - to provide access to storage engine functionality that works on the @@ -1892,9 +1945,48 @@ struct handlerton : public transaction_participant /********************************************************************* backup **********************************************************************/ + + /** BACKUP STAGE START */ void (*prepare_for_backup)(void); + /** BACKUP STAGE END */ void (*end_backup)(void); + /** + Start of a BACKUP SERVER phase, + when no backup_step() or backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to sink, or nullptr + @retval -1 on failure + */ + void *(*backup_start)(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink); + /** + Process a file that was collected in backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int (*backup_step)(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink); + /** + Finish a phase, once all calls for the current phase are completed. + @param thd current sesssion + @param target backup target + @param phase last phase on which backup_start() was successfully + invoked, or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success + */ + int (*backup_end)(THD *thd, const backup_target *target, backup_phase phase, + const backup_sink *sink); + /********************************************************************** WSREP specific **********************************************************************/ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 6766f3b0f273e..c3cc030c13dcc 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3543,6 +3543,7 @@ SHOW_VAR com_status_vars[]= { {"assign_to_keycache", STMT_STATUS(SQLCOM_ASSIGN_TO_KEYCACHE)}, {"backup", STMT_STATUS(SQLCOM_BACKUP)}, {"backup_lock", STMT_STATUS(SQLCOM_BACKUP_LOCK)}, + {"backup_server", STMT_STATUS(SQLCOM_BACKUP_SERVER)}, {"begin", STMT_STATUS(SQLCOM_BEGIN)}, {"binlog", STMT_STATUS(SQLCOM_BINLOG_BASE64_EVENT)}, {"call_procedure", STMT_STATUS(SQLCOM_CALL)}, diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc new file mode 100644 index 0000000000000..3351f8fdc3dcd --- /dev/null +++ b/sql/sql_backup.cc @@ -0,0 +1,838 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "mdl.h" +#include "mysys_err.h" +#include "sql_class.h" +#include "sql_backup.h" +#include "sql_backup_interface.h" +#include "sql_parse.h" +#include "my_atomic_wrapper.h" +#include "tpool.h" +#include "aligned.h" + +#if defined __linux__ || defined __FreeBSD__ +using copying_step= ssize_t(int,int,size_t,off_t*); +template +static ssize_t copy(int in_fd, int out_fd, off_t offset, off_t end) noexcept +{ + for (;;) + { + const size_t c{size_t(std::min(end - offset, INT_MAX >> 20 << 20))}; + ssize_t ret= step(in_fd, out_fd, c, &offset); + if (ret < 0) + { + if (nonblocking && errno == EAGAIN) + continue; + return ret; + } + if (offset == end) + return 0; + if (!ret) + return -1; + } +} + +# if 1 // disable to work around https://github.com/rr-debugger/rr/issues/4059 +/* Copy between files in a single (type of) file system */ +static inline ssize_t +copy_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return copy_file_range(in_fd, offset, out_fd, offset, count, 0); +} +# define cfr(src,dst,start,end) copy(src, dst, start, end) +# endif +#endif + +#ifdef _WIN32 +using tpool::pread; +using tpool::pwrite; +#else +# include +/** + Copy a file using a memory mapping. + @tparam stream true=write to a stream, false=pwrite to a file + @param in_fd source file + @param out_fd destination + @param o start offset + @param end last offset (exclusive) + @return error code + @retval 0 on success + @retval 1 if a memory mapping failed +*/ +template +static ssize_t mmap_copy(int in_fd, int out_fd, uint64_t o, uint64_t end) +{ +# if SIZEOF_SIZE_T < 8 + if (end != ssize_t(end)) + return 1; +# endif + const size_t count= size_t(end - o); + void *p= mmap(nullptr, count, PROT_READ, MAP_SHARED, in_fd, off_t(o)); + if (p == MAP_FAILED) + return 1; + ssize_t ret; + size_t c{count}; + for (const char *b= static_cast(p);; b+= ret, o+= uint64_t(ret)) + { + const size_t size{std::min(c, size_t(INT_MAX >> 20 << 20))}; + if (stream) + if ((ret= backup_stream_write(out_fd, b, size))) + break; + ret= stream ? size : pwrite(out_fd, b, size, off_t(o)); + if (ret < 0) + break; + c-= ret; + if (!c) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + munmap(p, c); + return ret; +} +#endif + +/** + Copy a file using positioned reads. + @tparam stream true=write to a stream, false=pwrite to a file + @param in_fd source file + @param out_fd destination + @param o start offset + @param end last offset (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +template +static ssize_t pread_write(IF_WIN(const native_file_handle&,int) in_fd, + IF_WIN(const native_file_handle&,int) out_fd, + uint64_t o, uint64_t end) + noexcept +{ + constexpr size_t READ_WRITE_SIZE= 65536; + char *b= static_cast(aligned_malloc(READ_WRITE_SIZE, 4096)); + if (!b) + return -1; + ssize_t ret; + for (uint64_t count{end - o};; o+= ret) + { + ret= pread(in_fd, b, + ssize_t(std::min(count, READ_WRITE_SIZE)), o); + if (ret > 0) + { + if (!stream) + ret= pwrite(out_fd, b, size_t(ret), o); + else if (backup_stream_write(out_fd, b, size_t(ret))) + { + ret= -1; + break; + } + } + if (ret < 0) + break; + count-= uint64_t(ret); + if (!count) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + aligned_free(b); + return ret; +} + +#ifdef __APPLE__ +/* The inline copy_entire_file() invokes fcopyfile() */ +#elif defined _WIN32 +/* CopyFileEx() should be used */ +#else +/** Copy a file (whole content). +@param src source file descriptor +@param dst target to append src to +@return error code (non-positive) +@retval 0 on success */ +extern "C" int copy_entire_file(int src, int dst) +{ + return copy_file(src, dst, 0, lseek(src, 0, SEEK_END)); +} +#endif + +/** Copy a portion of a file. +@param src source file descriptor +@param dst target to append src to +@param start first offset to copy +@param end last offset to copy (exclusive) +@return error code (non-positive) +@retval 0 on success */ +extern "C" int copy_file(IF_WIN(const native_file_handle&,int) src, + IF_WIN(const native_file_handle&,int) dst, + uint64_t start, uint64_t end) +{ + assert(end >= start); + ssize_t ret; +# ifdef cfr + if (!(ret= cfr(src, dst, off_t(start), off_t(end)))) + return int(ret); +# ifdef __linux__ + if (errno == EOPNOTSUPP || errno == EXDEV) +# endif +# endif +# ifdef __linux__ // starting with Linux 2.6.33, we can rely on sendfile(2) + return (start != 0 && off_t(start) != lseek(dst, start, SEEK_SET)) + ? -1 + : backup_stream_append_async(src, dst, start, end); +# else +# ifndef _WIN32 + if ((ret= mmap_copy(src, dst, start, end)) == 1) +# endif + ret= pread_write(src, dst, start, end); +# endif + assert(ret <= 0); + return int(ret); +} + +/** Append to the configuration file. +@param target backup target directory +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +extern "C" int backup_config_append(IF_WIN(const char*, int) target, + const char *config, size_t size) +{ + /* FIXME: append to a pre-created configuration file */ +#ifdef _WIN32 + HANDLE dst; + { + std::string path{target}; + path.append("/backup.cnf"); + dst= CreateFile(path.c_str(), GENERIC_WRITE, 0, + my_win_file_secattr(), CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (dst != INVALID_HANDLE_VALUE) + { + BOOL ok; + for (;;) + { + DWORD written; + ok= WriteFile(dst, config, DWORD(size), &written, nullptr); + if (ok || GetLastError() != ERROR_IO_PENDING) + break; + assert(written < DWORD(size)); + config+= written; + size-= size_t(written); + } + if (CloseHandle(dst) & ok) + return 0; + } + } +#else + int dst= openat(target, "backup.cnf", + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (dst < 0) + return dst; + ssize_t ret; + for (; (ret= write(dst, config, size)) >= 0; config+= ret, size -= ret) + { + assert(size_t(ret) <= size); + if (!(size-= size_t(ret))) + { + ret= 0; + break; + } + } + if (!(close(dst) | ret)) + return 0; +#endif + my_error(ER_CANT_CREATE_FILE, MYF(0), "backup.cnf", errno); + return -1; +} + +/** backup context */ +struct backup_target_phase +{ + /** target directory or stream */ + backup_target target; + /** current phase of backup */ + backup_phase phase; + /** backup worker state (output stream) */ + backup_sink sink; + /** stream object for streaming backup */ + FILE *stream; + /** handlerton::backup_step return value in multi-threaded operation */ + int ret; + /** engine-specific backup context */ + std::unordered_map &ha_data; +}; + +static my_bool backup_start(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + assert(int{t.phase} >= 0 || t.phase == BACKUP_PHASE_FINISH); + if (hton->backup_start) + { + t.sink.ha_data= t.ha_data[hton]; + void *data= hton->backup_start(thd, &t.target, t.phase, &t.sink); + if (data == reinterpret_cast(-1)) + return true; + assert(!t.ha_data[hton] || t.ha_data[hton] == data); + t.ha_data[hton]= data; + } + return false; +} + +static my_bool backup_end(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + if (hton->backup_end) + { + t.sink.ha_data= t.ha_data[hton]; + return hton->backup_end(thd, &t.target, t.phase, &t.sink); + } + return false; +} + +static my_bool backup_step(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + assert(int{t.phase} >= 0 || t.phase == BACKUP_PHASE_FINISH); + int res= 0; + if (hton->backup_step) + { + t.sink.ha_data= t.ha_data[hton]; + while ((res= hton->backup_step(thd, &t.target, t.phase, &t.sink))) + if (res < 0) + break; + } + return res != 0; +} + +/** Number of background tasks executing backup_step_callback */ +static Atomic_counter backup_step_callback_pending{0}; + +/** Invoke backup_step() in a background task */ +static void backup_step_callback(void *arg) noexcept +{ + backup_target_phase &t{*static_cast(arg)}; + assert(!t.ret); + t.ret= plugin_foreach_with_mask(nullptr, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, &t); +#ifndef NDEBUG + auto was_pending= +#endif + backup_step_callback_pending--; + assert(was_pending); +} + +/** + Execute all handlerton::backup_step() until completion or failure. + @param thd current connection + @param target_phase backup target and phase + @param threads number of execution threads + @param tp thread pool +*/ +static bool backup_steps(THD *thd, backup_target_phase *target_phase, + int threads, tpool::thread_pool *tp) +{ + assert(!backup_step_callback_pending); + if (threads == 1) + return plugin_foreach_with_mask(thd, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + tpool::task *const tasks= + static_cast(alloca(threads * sizeof *tasks)); + backup_step_callback_pending= threads - 1; + for (int n{threads}; --n; ) + { + target_phase[n].phase= target_phase->phase; + tp->submit_task(new (&tasks[n]) tpool::task{backup_step_callback, + &target_phase[n]}); + } + bool fail= plugin_foreach_with_mask(thd, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + while (backup_step_callback_pending) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + if (fail) + return fail; + + for (int n{threads}; --n; ) + if (target_phase[n].ret) + { + my_error(ER_UNKNOWN_ERROR, MYF(0)); + return true; + } + + return false; +} + +bool Sql_cmd_backup::execute(THD *thd) +{ + assert(!!target == !command); + + if (check_global_access(thd, RELOAD_ACL) || + check_global_access(thd, SELECT_ACL) || + (target && error_if_data_home_dir(target, "BACKUP SERVER TO"))) + return true; + + if (thd->current_backup_stage != BACKUP_FINISHED) + { + my_error(ER_BACKUP_LOCK_IS_ACTIVE, MYF(0)); + return true; + } + + /* Block concurrent BACKUP SERVER and BACKUP STAGE */ + MDL_request mdl_request; + MDL_REQUEST_INIT(&mdl_request, MDL_key::BACKUP, "", "", MDL_BACKUP_START, + MDL_EXPLICIT); + + if (thd->mdl_context.acquire_lock(&mdl_request, + thd->variables.lock_wait_timeout)) + return true; + + tpool::thread_pool *tp= nullptr; + std::unordered_map ha_data{}; + backup_target_phase *target_phase= static_cast + (alloca(threads * sizeof *target_phase)); + if (threads > 1 && !(tp= tpool::create_thread_pool_generic())) + { + oor: + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + err_exit: + thd->mdl_context.release_lock(mdl_request.ticket); + delete tp; + return true; + } + + if (command) + { + char cmd[1024]; + for (int t{threads}; t; ) + { + if (snprintf(cmd, sizeof cmd, "%s %d", command, t) >= int(sizeof cmd)) + goto oor; + FILE *f= my_popen(cmd, "w"); + if (!f) + { + while (t < threads) + my_pclose(target_phase[t++].stream); + goto oor; + } +#ifdef _WIN32 + HANDLE sink= (HANDLE) _get_osfhandle(_fileno(f)); +#else + int sink= fileno(f); +#endif + new (&target_phase[--t]) + backup_target_phase{backup_target{IF_WIN(nullptr, -1)}, + BACKUP_PHASE_START, backup_sink{sink, nullptr}, f, 0, ha_data}; + } + } + else if (my_mkdir(target, 0755, MYF(MY_WME))) + goto err_exit; + else + { +#ifndef _WIN32 + const int dir{open(target, O_DIRECTORY)}; + if (dir < 0) + { + my_error(EE_CANT_MKDIR, MYF(ME_BELL), target, errno); + goto err_exit; + } +#endif + for (int t{threads}; t; ) + { + new (&target_phase[--t]) + backup_target_phase{backup_target{IF_WIN(target, dir)}, + BACKUP_PHASE_START, + backup_sink{backup_sink::NO_STREAM, nullptr}, nullptr, 0, ha_data}; + } + } + + bool fail{false}; + + static_assert(int{MDL_BACKUP_START} + 1 == int{MDL_BACKUP_FLUSH}, ""); + static_assert(int{MDL_BACKUP_START} + 2 == int{MDL_BACKUP_WAIT_FLUSH}, ""); + static_assert(int{MDL_BACKUP_START} + 3 == int{MDL_BACKUP_WAIT_DDL}, ""); + static_assert(int{MDL_BACKUP_START} + 4 == int{MDL_BACKUP_WAIT_COMMIT}, ""); + static_assert(int{BACKUP_PHASE_START} + 1 == + int{BACKUP_PHASE_NO_BEGIN_NON_TRANS}, ""); + static_assert(int{BACKUP_PHASE_START} + 2 == + int{BACKUP_PHASE_NO_DML_NON_TRANS}, ""); + static_assert(int{BACKUP_PHASE_START} + 3 == int{BACKUP_PHASE_NO_DDL}, ""); + static_assert(int{BACKUP_PHASE_START} + 4 == + int{BACKUP_PHASE_NO_COMMIT}, ""); + int phase= int{BACKUP_PHASE_START}; + goto backup_phase_start; + + for (; phase <= int{BACKUP_PHASE_NO_COMMIT}; phase++) + { + assert(!fail); + { + const enum_mdl_type mdl= + enum_mdl_type(phase - int{BACKUP_PHASE_START} + int{MDL_BACKUP_START}); + fail= + thd->mdl_context.upgrade_shared_lock(mdl_request.ticket, mdl, + thd->variables.lock_wait_timeout); + if (fail) + break; + } + backup_phase_start: + target_phase->phase= backup_phase(phase); + fail= plugin_foreach_with_mask(thd, backup_start, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + if (fail) + break; + fail= backup_steps(thd, target_phase, threads, tp) || + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + if (fail) + break; + } + + /* The final part must not interfere with the use of the server datadir. + Release the locks. */ + thd->mdl_context.release_lock(mdl_request.ticket); + if (!fail) + { + target_phase->phase= BACKUP_PHASE_FINISH; + fail= plugin_foreach_with_mask(thd, backup_start, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase) || + backup_steps(thd, target_phase, threads, tp); + } + else + { + target_phase->phase= BACKUP_PHASE_ABORT; + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, target_phase); + target_phase->phase= BACKUP_PHASE_FINISH; + } + + fail= + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase) || fail; + delete tp; + + if (command) + for (int t= threads; t--; ) + my_pclose(target_phase[t].stream); +#ifndef _WIN32 + else + std::ignore= close(target_phase->target.fd); +#endif + + if (!fail) + my_ok(thd); + return fail; +} + +/** + Encode an octal string. + @param start first byte of buffer + @param end first byte after buffer + @param n number to encode +*/ +static void ustar_write_octal(char *start, char *end, uint64_t n) noexcept +{ + for (*--end= '\0'; --end >= start; n>>= 3) + *end= char('0' + (n & 7)); +} + +/** + Encode a quantity in 12 bytes. + @param start first byte of the buffer + @param n number to encode +*/ +static void ustar_write_dozen(char *start, uint64_t n) noexcept +{ + if (n < 1ULL << 33) + ustar_write_octal(start, start + 12, n); + else + { + const uint32_t head{my_htobe32(1U << 31)}; + n= my_htobe64(n); + memcpy(start, &head, 4); + memcpy(start + 4, &n, 8); + } +} + +/** Initialize a ustar block +@param buf GNU tape archiver --format=oldgnu header block +@param name name of the block +@param mode file access mode +@param size physical size of the following block */ +static void ustar_block_init(char *buf, const char *name, mode_t mode, + uint64_t size) noexcept +{ + strncpy(buf, name, 100); + ustar_write_octal(buf + 100, buf + 108, uint64_t(mode)); + ustar_write_octal(buf + 108, buf + 116, 0/* POSIX uid */); + ustar_write_octal(buf + 116, buf + 124, 0/* POSIX gid */); + ustar_write_dozen(buf + 124, size); + /* last modification time */ + ustar_write_octal(buf + 136, buf + 148, 0); + memset(buf + 148, ' ', 9); /* initial block checksum and dummy type */ + memset(buf + 157, '\0', 100); /* name of linked file (unused) */ + memcpy(buf + 257, "ustar ", 8); + strncpy(buf + 265, "root" /* POSIX owner name */, 32); + strncpy(buf + 297, "root" /* POSIX group name */, 512 - 297); + /* The caller will fill in the rest and invoke ustar_block_checksum() */ +} + +/** + Compute and write the POSIX tar block checksum. + @param buf POSIX tar block +*/ +static void ustar_block_checksum(char *buf) noexcept +{ + uint16_t sum{0}; + for (int i{0}; i < 512; i++) + sum+= uint16_t{uint8_t(buf[i])}; + ustar_write_octal(buf + 148, buf + 155, sum); +} + +/** + Write data to a stream. + @param stream backup stream + @param buf source buffer + @param size length of the buffer (usually an integer multiple of 512) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_write(IF_WIN(HANDLE, int) stream, const void *buf, + size_t size) +{ +#ifdef _WIN32 + for (DWORD sz= DWORD(size);;) + { + DWORD wrote; + if (WriteFile(stream, buf, sz, &wrote, nullptr)) + { + assert(wrote == sz); + return 0; + } + else if (GetLastError() != ERROR_IO_PENDING) + return -1; + buf= static_cast(buf) + wrote; + sz-= wrote; + } +#else + do + { + ssize_t wrote= write(stream, buf, size); + assert(wrote <= ssize_t(size)); + if (wrote < 0) + { + if (errno == EAGAIN) + continue; + return -1; + } + buf= static_cast(buf) + wrote; + size-= size_t(wrote); + } + while (size); +#endif + return 0; +} + +/** + Copy a prefix of a NUL terminated string to a buffer, NUL-padded. + @param b output buffer + @param s NUL terminated string + @param size size of buf, in bytes +*/ +static inline char *ustar_zeropad(char *b, const char *s, size_t size) noexcept +{ +#if defined __GNUC__ && __GNUC__ >= 8 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wstringop-truncation" +#endif + return strncpy(b, s, size); +#if defined __GNUC__ && __GNUC__ >= 8 +# pragma GCC diagnostic pop +#endif +} + +/** Start streaming a file. +@param stream backup stream +@param name file name +@param mode file access mode +@param size physical length of the file, in bytes +@param chunks payload chunks of a sparse file, or nullptr +@param n_chunks number of chunks; 0 unless sparse file +@return error code (non-positive) +@retval 0 on success */ +extern "C" +int backup_stream_start(IF_WIN(HANDLE, int) stream, + const char *name, mode_t mode, uint64_t size, + const struct backup_chunk *chunks, size_t n_chunks) +{ + assert(stream != backup_sink::NO_STREAM); + char buf[512]; + size_t s= strlen(name); + if (s > 100) + { + /* Write a block that contains the full name length, + followed by blocks that contain the full name, in + tar --format=oldgnu */ + ustar_block_init(buf, "././@LongLink", 0644, s); + ustar_block_checksum(buf); + if (int err= backup_stream_write(stream, buf, sizeof buf)) + return err; + const size_t whole{s & ~(sizeof buf)}; + if (whole) + if (int err= backup_stream_write(stream, name, whole)) + return err; + if (s - whole) + { + ustar_zeropad(buf, name + whole, sizeof buf); + if (int err= backup_stream_write(stream, buf, sizeof buf)) + return err; + } + } + + ustar_block_init(buf, name, mode, size); + if (!n_chunks) + buf[156]= '0'; + else + { + buf[156]= 'S'; + char *h= &buf[386]; + if (n_chunks > 4) + return -1; // FIXME; support more chunks + for (size_t i= 0; i < n_chunks; i++, h+= 24) + { + ustar_write_dozen(h, chunks[i].offset); + ustar_write_dozen(h + 12, chunks[i].length); + } + ustar_write_dozen(&buf[0x1e3], chunks[n_chunks - 1].offset + + chunks[n_chunks - 1].length); + } + ustar_block_checksum(buf); + return backup_stream_write(stream, buf, sizeof buf); +} + +/** Append to the configuration file. +@param target backup stream +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +extern "C" int backup_stream_config(IF_WIN(HANDLE, int) stream, + const char *config, size_t size) +{ + /* FIXME: append to a pre-created configuration file */ + if (int ret= + backup_stream_start(stream, "backup.cnf", 0644, size, nullptr, 0)) + return ret; + char buf[512]; + const size_t whole{size & ~((sizeof buf) - 1)}; + if (whole) + if (int err= backup_stream_write(stream, config, whole)) + return err; + if (size == whole) + return 0; + ustar_zeropad(buf, config + whole, sizeof buf); + return backup_stream_write(stream, buf, sizeof buf); +} + +/** + Append a file snippet to stream, + after a corresponding call to backup_stream_start(). + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_append(IF_WIN(const native_file_handle&,int) src, + IF_WIN(HANDLE, int) stream, + uint64_t start, uint64_t end) +{ + assert(stream != backup_sink::NO_STREAM); + ssize_t ret; +#ifndef _WIN32 + if ((ret= mmap_copy(src, stream, start, end)) == 1) +#endif + ret= pread_write(src, stream, start, end); + return int(ret); +} + +#ifdef __linux__ +# include +/** Copy a file to a stream or to a regular file. */ +static inline ssize_t +send_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return sendfile(out_fd, in_fd, offset, count); +} + +/** + Append an immutable snippet of a file to the stream, + allowing Linux sendfile(2) to be invoked. + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_append_async(int src, int stream, + uint64_t start, uint64_t end) +{ + assert(stream != backup_sink::NO_STREAM); + return int(copy(src, stream, off_t(start), off_t(end))); +} +#endif + +#ifdef _WIN32 +extern "C" int backup_stream_append_plain(HANDLE src, HANDLE stream, + uint64_t start, uint64_t end) +{ + return backup_stream_append(src, stream, start, end); +} +#endif diff --git a/sql/sql_backup.h b/sql/sql_backup.h new file mode 100644 index 0000000000000..2759e0d64b43d --- /dev/null +++ b/sql/sql_backup.h @@ -0,0 +1,53 @@ +/***************************************************************************** +Copyright (c) 2026 MariaDB plc. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +#pragma once + +/** BACKUP SERVER */ +class Sql_cmd_backup : public Sql_cmd +{ + /** target directory, or nullptr when streaming */ + const char *const target{nullptr}; + /** argument of my_popen() for streaming backup, or nullptr */ + const char *const command{nullptr}; + /** number of concurrent threads to use */ + const int threads; + +public: + /** + Constructor. + @param target name of target or scratch directory + @param threads number of concurrent threads to use + */ + Sql_cmd_backup(LEX_CSTRING target, int threads) : + target(target.str), threads(threads) {} + /** + Constructor. + @param threads number of concurrent threads to use + @param command nullptr, or a shell command for handling a backup stream + */ + Sql_cmd_backup(int threads, LEX_CSTRING command) : + command(command.str), threads(threads) {} + ~Sql_cmd_backup()= default; + + bool execute(THD *thd) override; + + enum_sql_command sql_command_code() const override + { + return SQLCOM_BACKUP_SERVER; + } +}; diff --git a/sql/sql_backup_interface.h b/sql/sql_backup_interface.h new file mode 100644 index 0000000000000..94892ba7591ab --- /dev/null +++ b/sql/sql_backup_interface.h @@ -0,0 +1,178 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +struct backup_target; + +/** A payload chunk in a sparse file that is being streamed */ +struct backup_chunk +{ + /** byte offset of the start of the payload, from the start of the file */ + uint64_t offset; + /** length of the hole */ + uint64_t length; +}; + +#ifdef _WIN32 +/* Use CopyFileEx() to copy entire files */ +struct native_file_handle; +#elif defined __APPLE__ +/* You should invoke fclonefileat(2) manually before attempting +copy_entire_file() or copy_file() */ +# include +# include +# include +/** Copy an entire file. +@param src source file descriptor +@param dst target to append src to +@return error code (negative) +@retval 0 on success */ +inline int copy_entire_file(int src, int dst) +{ + return fcopyfile(src, dst, NULL, COPYFILE_ALL | COPYFILE_CLONE); +} +#else +# ifdef __cplusplus +extern "C" +# endif +/** Copy an entire file. +@param src source file descriptor +@param dst target to append src to +@return error code (non-positive) +@retval 0 on success */ +int copy_entire_file(int src, int dst); +#endif + +#ifdef __cplusplus +extern "C" +#endif +/** Copy a portion of a file. +@param src source file descriptor +@param dst target to append src to +@param start first offset to copy +@param end last offset to copy (exclusive) +@return error code (non-positive) +@retval 0 on success */ +int copy_file(IF_WIN(const native_file_handle&,int) src, + IF_WIN(const native_file_handle&,int) dst, + uint64_t start, uint64_t end); + +#ifdef __cplusplus +extern "C" +#endif +/** Append to the configuration file. +@param target backup target directory +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +int backup_config_append(IF_WIN(const char*, int) target, + const char *config, size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** Append to the configuration file. +@param target backup stream +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +int backup_stream_config(IF_WIN(HANDLE, int) stream, + const char *config, size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** Start streaming a file. +@param target backup target +@param name file name +@param mode file access mode +@param size physical length of the file, in bytes +@param chunks payload chunks of a sparse file, or nullptr +@param n_chunks number of chunks; 0 unless sparse file +@return error code (non-positive) +@retval 0 on success */ +int backup_stream_start(IF_WIN(HANDLE, int) stream, + const char *name, mode_t mode, uint64_t size, + const struct backup_chunk *chunks, size_t n_chunks); + +#ifdef __cplusplus +extern "C" +#endif +/** + Write data to a stream. + @param stream backup stream + @param buf source buffer + @param size length of the buffer (usually an integer multiple of 512) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_write(IF_WIN(HANDLE, int) stream, const void *buf, + size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** + Append a file snippet to the stream, + after a corresponding call to backup_stream_start(). + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_append(IF_WIN(const native_file_handle&,int) src, + IF_WIN(HANDLE, int) stream, + uint64_t start, uint64_t end); + +#ifdef __linux__ +# ifdef __cplusplus +extern "C" +# endif +/** + Append an immutable snippet of a file to the stream, + allowing Linux sendfile(2) to be invoked. + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_append_async(int src, int stream, + uint64_t start, uint64_t end); +#elif defined _WIN32 +# define backup_stream_append_async backup_stream_append_plain +#else +# define backup_stream_append_async backup_stream_append +#endif + +#ifdef _WIN32 +# ifdef __cplusplus +extern "C" +# endif +int backup_stream_append_plain(HANDLE src, HANDLE stream, + uint64_t start, uint64_t end); +#else +# define backup_stream_append_plain backup_stream_append +#endif diff --git a/sql/sql_command.h b/sql/sql_command.h index 9c9166706a034..b8903399711f0 100644 --- a/sql/sql_command.h +++ b/sql/sql_command.h @@ -103,6 +103,7 @@ enum enum_sql_command { SQLCOM_SHOW_PACKAGE_BODY_CODE, SQLCOM_BACKUP, SQLCOM_BACKUP_LOCK, SQLCOM_SHOW_CREATE_SERVER, + SQLCOM_BACKUP_SERVER, /* When a command is added here, be sure it's also added in mysqld.cc diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index bcd6de564f550..b549017622332 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -781,6 +781,7 @@ void init_update_queries(void) sql_command_flags[SQLCOM_DROP_SERVER]|= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP]= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP_LOCK]= CF_AUTO_COMMIT_TRANS; + sql_command_flags[SQLCOM_BACKUP_SERVER]= CF_AUTO_COMMIT_TRANS; /* The following statements can deal with temporary tables, @@ -5899,6 +5900,7 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt) case SQLCOM_CALL: case SQLCOM_REVOKE: case SQLCOM_GRANT: + case SQLCOM_BACKUP_SERVER: if (thd->variables.option_bits & OPTION_IF_EXISTS) lex->create_info.set(DDL_options_st::OPT_IF_EXISTS); DBUG_ASSERT(lex->m_sql_cmd != NULL); @@ -10254,7 +10256,7 @@ int test_if_data_home_dir(const char *dir) if (!dir) DBUG_RETURN(0); - (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH); + (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH|MY_RESOLVE_SYMLINKS); DBUG_RETURN(path_starts_from_data_home_dir(path)); } diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 56d849306556c..04907ca9d3140 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -50,6 +50,7 @@ #include "sql_alter.h" // Sql_cmd_alter_table* #include "sql_truncate.h" // Sql_cmd_truncate_table #include "sql_admin.h" // Sql_cmd_analyze/Check..._table +#include "sql_backup.h" #include "sql_partition_admin.h" // Sql_cmd_alter_table_*_part. #include "sql_handler.h" // Sql_cmd_handler_* #include "sql_signal.h" @@ -1472,7 +1473,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %type json_on_response %type json_type_constraint -%type json_key_unique_constraint +%type json_key_unique_constraint opt_concurrent %type json_predicate %type field_type field_type_all field_type_all_builtin @@ -15562,6 +15563,31 @@ backup_statements: /* Table list is empty for unlock */ Lex->sql_command= SQLCOM_BACKUP_LOCK; } + | SERVER_SYM TO_SYM TEXT_STRING_sys opt_concurrent + { + Lex->sql_command= SQLCOM_BACKUP_SERVER; + Lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_backup($3, $4); + } + | SERVER_SYM WITH opt_concurrent TEXT_STRING_sys + { + Lex->sql_command= SQLCOM_BACKUP_SERVER; + Lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_backup($3, $4); + } + ; + +opt_concurrent: + /* empty */ + { $$= 1; } + | ulonglong_num CONCURRENT + { + $$= int($1); + if ($1 < 1 || $1 > 256) + { + my_error(ER_DATA_OUT_OF_RANGE, myf(0), "CONCURRENT", + "BACKUP SERVER"); + MYSQL_YYABORT; + } + } ; opt_delete_gtid_domain: diff --git a/sql/sys_vars.inl b/sql/sys_vars.inl index 0f8aa1eb63bf6..8e5f0b984e81e 100644 --- a/sql/sys_vars.inl +++ b/sql/sys_vars.inl @@ -2506,6 +2506,7 @@ public: bool session_update(THD *thd, set_var *var) override; }; +#ifdef HAVE_REPLICATION /* Class for replicate_events_marked_for_skip. We need a custom update function that ensures the slave is stopped when @@ -2647,6 +2648,7 @@ public: } const uchar *global_value_ptr(THD *thd, const LEX_CSTRING *base) const override; }; +#endif /* HAVE_REPLICATION */ /** diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 9e3a23b34ab46..d63751a16af08 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -185,6 +185,8 @@ SET(INNOBASE_SOURCES handler/handler0alter.cc handler/innodb_binlog.cc handler/i_s.cc + handler/backup_innodb.h + handler/backup_innodb.cc ibuf/ibuf0ibuf.cc include/btr0btr.h include/btr0btr.inl diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 756a8b85d4752..50fb2f69d4a25 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -35,6 +35,7 @@ Created 11/11/1995 Heikki Tuuri #include "buf0buf.h" #include "buf0checksum.h" #include "buf0dblwr.h" +#include "backup_innodb.h" #include "srv0start.h" #include "page0zip.h" #include "fil0fil.h" @@ -1000,6 +1001,13 @@ uint32_t fil_space_t::flush_freed(bool writable) noexcept mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex); mysql_mutex_assert_not_owner(&buf_pool.mutex); + /* Note: There is no need to invoke writing_start() or + writing_stop() here, because we are only overwriting freed (garbage) + pages. If backup reads a torn page, it will also have copied a + corresponding FREE_PAGE record, which would be applied on recovery. + Besides, the freed page should never be reachable from other pages + that are part of the snapshot. */ + const bool punch_hole= chain.start->punch_hole == 1; if (!punch_hole && !srv_immediate_scrub_data_uncompressed) return 0; @@ -1275,6 +1283,16 @@ ATTRIBUTE_COLD static size_t buf_flush_LRU_to_withdraw(size_t to_withdraw, return to_withdraw; } +/** Stop writing to a tablespace. +@param space tablespace +@return nullptr */ +static fil_space_t *writing_stop(fil_space_t *space) noexcept +{ + space->writing_stop(); + space->release(); + return nullptr; +} + /** Flush dirty blocks from the end buf_pool.LRU, and move clean blocks to buf_pool.free. @param max maximum number of blocks to flush @@ -1292,6 +1310,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1371,7 +1390,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, buf_pool.lru_hp.set(bpage); mysql_mutex_unlock(&buf_pool.mutex); if (space) - space->release(); + writing_stop(space); auto p= buf_flush_space(space_id); space= p.first; last_space_id= space_id; @@ -1380,6 +1399,10 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, mysql_mutex_lock(&buf_pool.mutex); goto no_space; } + + backup_page_end= space->writing_start() + ? space->backup_page_end() : 0; + mysql_mutex_lock(&buf_pool.mutex); buf_pool.stat.n_pages_written+= p.second; } @@ -1391,8 +1414,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, } else if (space->is_stopping_writes()) { - space->release(); - space= nullptr; + space= writing_stop(space); no_space: mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_discard_page(bpage); @@ -1409,7 +1431,8 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, break; } - if (neighbors && space->is_rotational() && UNIV_LIKELY(!to_withdraw) && + if (neighbors && UNIV_LIKELY(!(to_withdraw | backup_page_end)) && + space->is_rotational() && /* Skip neighbourhood flush from LRU list if we haven't yet reached half of the free page target. */ UT_LIST_GET_LEN(buf_pool.free) * 2 >= free_limit) @@ -1421,10 +1444,17 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, flush: if (UNIV_UNLIKELY(to_withdraw != 0)) to_withdraw= buf_flush_LRU_to_withdraw(to_withdraw, *bpage); - if (bpage->flush(space)) + const uint32_t page{bpage->id().page_no()}; + if (page < backup_page_end && + page >= backup_page_end - space->BACKUP_BATCH_SIZE) + bpage->lock.u_unlock(true); + else if (bpage->flush(space)) + { ++n->flushed; - else - continue; + goto reacquire_mutex; + } + + continue; } goto reacquire_mutex; @@ -1437,7 +1467,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, buf_pool.lru_hp.set(nullptr); if (space) - space->release(); + writing_stop(space); if (scanned) { @@ -1484,6 +1514,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1555,10 +1586,12 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); mysql_mutex_unlock(&buf_pool.mutex); if (space) - space->release(); + writing_stop(space); auto p= buf_flush_space(space_id); space= p.first; last_space_id= space_id; + backup_page_end= space && space->writing_start() + ? space->backup_page_end() : 0; mysql_mutex_lock(&buf_pool.mutex); buf_pool.stat.n_pages_written+= p.second; mysql_mutex_lock(&buf_pool.flush_list_mutex); @@ -1567,10 +1600,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept ut_ad(!space); } else if (space->is_stopping_writes()) - { - space->release(); - space= nullptr; - } + space= writing_stop(space); if (!space) buf_flush_discard_page(bpage); @@ -1579,9 +1609,17 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); do { - if (neighbors && space->is_rotational()) + if (neighbors && UNIV_LIKELY(!backup_page_end) && + space->is_rotational()) count+= buf_flush_try_neighbors(space, page_id, bpage, neighbors == 1, count, max_n); + else if (page_id.page_no() < backup_page_end && + page_id.page_no() >= + backup_page_end - space->BACKUP_BATCH_SIZE) + { + bpage->lock.u_unlock(true); + continue; + } else if (bpage->flush(space)) ++count; else @@ -1600,7 +1638,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept buf_pool.flush_hp.set(nullptr); if (space) - space->release(); + writing_stop(space); if (scanned) { @@ -1691,6 +1729,7 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept if (written) buf_pool.stat.n_pages_written+= written; } + mysql_mutex_lock(&buf_pool.flush_list_mutex); for (buf_page_t *bpage= UT_LIST_GET_LAST(buf_pool.flush_list); bpage; ) @@ -1733,17 +1772,35 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept acquired= false; goto was_freed; } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (bpage->flush(space)) + uint32_t page, backup_page_end; + + if (UNIV_UNLIKELY(space->writing_start())) { - ++n_flush; - if (!--max_n_flush) + page= bpage->id().page_no(); + backup_page_end= space->backup_page_end(); + if (page < backup_page_end && + page >= backup_page_end - space->BACKUP_BATCH_SIZE) { + bpage->lock.u_unlock(true); + space->writing_stop(); + skip: mysql_mutex_lock(&buf_pool.mutex); mysql_mutex_lock(&buf_pool.flush_list_mutex); may_have_skipped= true; goto done; } + } + + const bool written{bpage->flush(space)}; + space->writing_stop(); + + if (written) + { + ++n_flush; + if (!--max_n_flush) + goto skip; mysql_mutex_lock(&buf_pool.mutex); } } @@ -2060,6 +2117,7 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept this->end_lsn= end_lsn; if (!archive) { + archived_checkpoint= checkpoint; archived_lsn= end_lsn; checkpoint_completed: if (resize_log.m_file == log.m_file) @@ -2077,7 +2135,7 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept /* Make the previous archived log file read-only */ #ifdef _WIN32 resize_log.close(); - SetFileAttributesA(get_archive_path().c_str(), + SetFileAttributesA(get_archive_path(get_first_lsn() - capacity()).c_str(), FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_ARCHIVE); #else struct stat st; @@ -2087,9 +2145,10 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept st.st_mode= 0444; if (fchmod(resize_log.m_file, st.st_mode)) my_error(ER_ERROR_ON_CLOSE, MYF(ME_ERROR_LOG), - get_archive_path().c_str(), errno); + get_archive_path(get_first_lsn() - capacity()).c_str(), errno); resize_log.close(); #endif + innodb_backup_checkpoint(); } else goto checkpoint_completed; diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc new file mode 100644 index 0000000000000..558c9b3f36f4c --- /dev/null +++ b/storage/innobase/handler/backup_innodb.cc @@ -0,0 +1,1110 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "sql_class.h" +#include "backup_innodb.h" +#include "sql_backup_interface.h" +#include "trx0trx.h" +#include "buf0flu.h" +#include "log0crypt.h" +#include +#ifdef __linux__ +# include +# include +#endif + +/** Associate a transaction with the current session +@param thd session +@return InnoDB transaction */ +trx_t *check_trx_exists(THD *thd) noexcept; + +namespace +{ +/** Backup state; protected by log_sys.latch */ +class InnoDB_backup +{ + /** Backup context */ + struct context + { + /** Start LSN of the first backed up log file */ + lsn_t first_lsn; + /** Start LSN of the last log file, or LSN_MAX if not determined yet */ + lsn_t max_first_lsn; + /** Final LSN of the backup, or LSN_MAX if not determined yet */ + lsn_t last_lsn; + /** size of the first log file */ + uint64_t first_size; + /** Checkpoint at the start of the backup */ + lsn_t checkpoint; + /** Log record pointing to the checkpoint */ + lsn_t checkpoint_end_lsn; + /** the original state of innodb_log_archive before/after backup */ + bool archived; + /** whether cleanup() was invoked */ + bool cleaned_up; + /** the start LSN of the last hard-linked file, or 0 */ + std::atomic last_hardlink; + + /** + Note that a log file was hard-linked. + @param lsn start LSN of a hard-linked file + */ + void note_hardlink(lsn_t lsn) noexcept + { + for (lsn_t last= last_hardlink.load(std::memory_order_relaxed); + last < lsn && !last_hardlink. + compare_exchange_weak(last, lsn, + std::memory_order_relaxed, + std::memory_order_relaxed); ) {} + } + + /** Ensure that the last, hard-linked log file is not shared with + the server data directory, by copying it until the final LSN + @param target backup target directory + @param hl last_hardlink + @return error code + @retval 0 on success + */ + ATTRIBUTE_COLD int de_hardlink(const backup_target &target, lsn_t hl) + noexcept + { +#ifdef _WIN32 + std::string src{target.path}; + src.push_back('/'); + std::string dst{src}; + src.append("ib_logfile101"); + log_sys.append_archive_name(dst, hl); + const char *const s_{src.c_str()}, *const d_{dst.c_str()}; + if (!MoveFileEx(d_, s_, 0)) + { + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), d_, s_, errno); + return 1; + } + HANDLE s, d; + for (;;) + { + s= CreateFile(s_, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (s != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + my_osmaperr(GetLastError()); + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), s_, errno); + return 1; + } + d= CreateFile(d_, GENERIC_WRITE, 0, my_win_file_secattr(), + CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); + if (d == INVALID_HANDLE_VALUE) + { + fail: + my_osmaperr(GetLastError()); + std::ignore= CloseHandle(s); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), s_, d_, errno); + return 1; + } + const uint64_t payload_end{log_sys.START_OFFSET + last_lsn - hl}; + /* First, extend the file to a valid size. */ + int err; + { + LARGE_INTEGER li; + li.QuadPart= std::max(log_sys.FILE_SIZE_MIN, + (payload_end + 4095) & ~4095ULL); + err= !SetFilePointerEx(d, li, nullptr, FILE_BEGIN) || !SetEndOfFile(d); + } + if (!err && !(err= copy_file(s, d, log_sys.START_OFFSET, payload_end)) && + hl == first_lsn) + { + uint64_t cp_buf[8]{}; + write_checkpoint_buf(cp_buf, + checkpoint_end_lsn - hl + log_sys.START_OFFSET); + err= write_checkpoint(d, cp_buf); + } + if (!CloseHandle(d) | err) + goto fail; + + std::ignore= CloseHandle(s); + + if (!DeleteFile(s_)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_DELETE_FILE, MYF(ME_ERROR_LOG), s_, errno); + return 1; + } +#else + std::string dst; + log_sys.append_archive_name(dst, hl); + const char *const d_{dst.c_str()}; + int d{-1}; + int err= ER_FILE_NOT_FOUND; + int s= openat(target.fd, d_, O_RDONLY); + if (s == -1) + { + error_return: + my_error(err, MYF(ME_ERROR_LOG), d_, errno); + if (s != -1) + std::ignore= close(s); + return 1; + } + err= ER_CANT_DELETE_FILE; + if (unlinkat(target.fd, d_, 0)) + goto error_return; + err= ER_CANT_CREATE_FILE; + d= openat(target.fd, d_, O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (d < 0) + goto error_return; + const uint64_t payload_end{log_sys.START_OFFSET + last_lsn - hl}; + /* First, extend the file to a valid size. */ + int fail= + ftruncate(d, std::max(log_sys.FILE_SIZE_MIN, + (payload_end + 4095) & ~4095LL)) || + copy_file(s, d, log_sys.START_OFFSET, payload_end); + if (!fail && hl == first_lsn) + { + uint64_t cp_buf[8]{}; + write_checkpoint_buf(cp_buf, + checkpoint_end_lsn - hl + log_sys.START_OFFSET); + fail= write_checkpoint(d, cp_buf); + } + if (close(d) | fail) + goto error_return; + std::ignore= close(s); +#endif + return 0; + } + + /** + Finish a backup. + @param target backup target + @param sink backup worker context + @return error code + @retval 0 on success + */ + int cleanup(const backup_target &target, const backup_sink &sink) noexcept + { + const lsn_t hl{last_hardlink.load(std::memory_order_relaxed)}; + if (hl == LSN_MAX) + return 0; + log_sys.latch.rd_lock(); + const lsn_t current_first_lsn{log_sys.get_first_lsn()}; + log_sys.latch.rd_unlock(); + if (hl == current_first_lsn) + { + ut_ad(sink.stream == sink.NO_STREAM); + if (int fail= de_hardlink(target, hl)) + return fail; + } + return write_config(target, sink); + } + }; + + /** pointer to backup context, or nullptr if no backup is active */ + context *ctx; + + /** the original innodb_log_file_size, or 0 */ + uint64_t old_size; + + /** collection of files to be copied */ + std::vector queue; + /** collection of completed log archive files to be + hard-linked, copied, or moved */ + std::vector logs; + +public: + /** + Start of BACKUP SERVER: collect all files to be backed up + @param thd current session + @return ctx + @retval -1 on failure + */ + void *init(THD *thd) noexcept + { + log_sys.latch.wr_lock(); + ut_ad(!ctx); + ut_ad(queue.empty()); + if (!logs.empty()) + { + /* A new BACKUP SERVER is being invoked before a previous one + had been fully finalized. Clean up any log files. */ + if (old_size) + delete_logs(); + logs.clear(); + } + + const bool fail{log_sys.backup_start(&old_size, thd)}; + + if (!fail) + { + lsn_t start_end; + const lsn_t start= +#if 1 /* TODO: for incremental backup, allow the start to be specified */ + log_sys.get_latest_checkpoint(start_end); +#else + log_sys.archived_checkpoint; + start_end= log_sys.archived_lsn; +#endif + ctx= new context{ + log_sys.get_first_lsn(), LSN_MAX, LSN_MAX, log_sys.file_size, + start, start_end, !old_size, false, 0 + }; + + /* Collect all tablespaces that have been created before our + start checkpoint. Newer tablespaces will be recovered by the + innodb_log_archive=ON recovery. + + If a tablespace is deleted before step() is invoked, the file + will not be copied, and a FILE_DELETE record in the log will + ensure correct recovery. + + If a tablespace is renamed between this and end(), the recovery + of a FILE_RENAME record will ensure the correct file name, + no matter which name was used by step(). */ + mysql_mutex_lock(&fil_system.mutex); + for (fil_space_t &space : fil_system.space_list) + if (space.id < SRV_SPACE_ID_UPPER_BOUND && + !space.is_being_imported() && + /* FIXME: how to initialize create_lsn for old files, to + have efficient incremental backup? + fil_node_t::read_page0() cannot assign it from + FIL_PAGE_LSN because that would not reflect the file + creation but for example allocating or freeing a page. + + The easy parts of initializing space->create_lsn are + as follows: + (1) In log_parse_file() when processing FILE_CREATE + (2) In deferred_spaces.create() */ + space.get_create_lsn() < start) + queue.emplace_back(space.id); + mysql_mutex_unlock(&fil_system.mutex); + } + log_sys.latch.wr_unlock(); + DEBUG_SYNC(thd, "innodb_backup_start"); + return fail ? reinterpret_cast(-1) : ctx; + } + + /** + Process a file that was collected at init(). + This may be invoked from multiple concurrent threads. + @param target backup target + @param phase backup phase + @param sink backup worker context + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int step(const backup_target &target, backup_phase phase, + const backup_sink &sink) noexcept + { + uint32_t id{FIL_NULL}; + lsn_t lsn{0}; + log_sys.latch.wr_lock(); + const lsn_t first{log_sys.get_first_lsn()}; + ut_ad(sink.ha_data); + ut_ad(ctx ? ctx == sink.ha_data + : phase == BACKUP_PHASE_FINISH || phase == BACKUP_PHASE_NO_COMMIT); + ut_ad(static_cast(sink.ha_data)->last_lsn == LSN_MAX + ? phase == BACKUP_PHASE_START : !ctx); + size_t size{queue.size()}; + ut_ad(!size || phase == BACKUP_PHASE_START); + if (!logs.empty()) + { + lsn= logs.back(); + logs.pop_back(); + if (!size) + size= logs.size(); + } + else if (size) + { + ut_ad(phase == BACKUP_PHASE_START); + size--; + id= queue.back(); + queue.pop_back(); + } + log_sys.latch.wr_unlock(); + + if (lsn) + { + if (UNIV_UNLIKELY(lsn > first)) + /* Wait for checkpoint_complete(). */ + buf_flush_sync_batch(lsn, true); + if (replicate(lsn, target, sink, lsn < first)) + return -1; + } + else if (fil_space_t *space= fil_space_t::get(id)) + { + int res= -1; + uint32_t start{0}; +#ifdef _WIN32 + if (sink.stream == sink.NO_STREAM) + { + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain); node; + start+= node->size, node= UT_LIST_GET_NEXT(chain, node)) + if ((res= backup(target.path, node, start))) + break; + } + else + { + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain); node; + start+= node->size, node= UT_LIST_GET_NEXT(chain, node)) + if ((res= stream(sink.stream, node, start))) + break; + } +#else + int fd; + int (*method)(int, fil_node_t *, uint32_t start); + if (sink.stream == sink.NO_STREAM) + { + fd= target.fd; + method= backup; + } + else + { + fd= sink.stream; + method= stream; + } + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain); node; + start+= node->size, node= UT_LIST_GET_NEXT(chain, node)) + if ((res= (*method)(fd, node, start))) + break; +#endif + space->release(); + if (res) + return res; + } + + size= std::min(size_t{std::numeric_limits::max()}, size); + return int(size); + } + + /** + Determine the logical time of the backup snapshot. + */ + void commit() noexcept + { + log_sys.latch.wr_lock(); + ut_ad(queue.empty()); + ut_ad(ctx); + ut_ad(ctx->last_lsn == LSN_MAX); + const lsn_t last_lsn{log_sys.get_lsn()}; + lsn_t lsn{log_sys.get_first_lsn()}; + if (logs.empty() || logs.back() != lsn) + { + /* Schedule the remaining log for copying */ + logs.emplace_back(lsn); + const lsn_t next_lsn{lsn + log_sys.capacity()}; + if (next_lsn < last_lsn) + logs.emplace_back(lsn= next_lsn); + } + ctx->max_first_lsn= lsn; + ctx->last_lsn= last_lsn; + ctx= nullptr; /* unsubscribe to checkpoint_complete() */ + log_sys.latch.wr_unlock(); + } + + /** + Finish copying or finalize the backup. + @param thd current session + @param phase backup phase + @param sink backup worker context + @return error code + @retval 0 on success + */ + int end(THD *thd, backup_phase phase, const backup_sink &sink) noexcept + { + context *const ctx{static_cast(sink.ha_data)}; + if (ctx->cleaned_up) + return 0; + ctx->cleaned_up= true; + if (phase == BACKUP_PHASE_ABORT) + ctx->last_hardlink.store(LSN_MAX, std::memory_order_relaxed); + log_sys.latch.wr_lock(); + ut_ad(!this->ctx || this->ctx == ctx); + this->ctx= nullptr; /* fini() will delete the object */ + ut_ad(!log_sys.resize_in_progress()); + ut_ad(log_sys.archive); + queue.clear(); + cleanup(); + + int fail{0}; + + if (old_size) + { + log_sys.latch.wr_unlock(); + fail= log_sys.backup_stop_archiving(thd); + log_sys.latch.wr_lock(); + } + + log_sys.backup_stop(old_size, thd); + return fail; + } + + /** + Clean up after end(). + @param target backup target + @param sink backup worker context + @return error code + @retval 0 on success + */ + int fini(const backup_target &target, const backup_sink &sink) noexcept + { + if (context *ctx{static_cast(sink.ha_data)}) + { + ut_ad(ctx != this->ctx); + int fail{ctx->cleanup(target, sink)}; + delete ctx; + return fail; + } + return 0; + } + + /** + Complete the first checkpoint in a new archive log file. + */ + void checkpoint_complete() noexcept + { + ut_ad(log_sys.latch_have_wr()); + if (ctx) + logs.emplace_back(log_sys.get_first_lsn() - log_sys.capacity()); + } + +private: + /* Clean up resources at the end of backup. */ + void cleanup() noexcept + { + if (old_size) + delete_logs(); + logs.clear(); + } + + /** Safely start backing up a tablespace file + @param end last page to copy */ + static void backup_batch_start(fil_space_t *space, uint32_t end) noexcept + { + if (space->backup_start(end)) + os_aio_wait_until_no_pending_writes(false); + } + /* Stop backing up a tablespace */ + static void backup_batch_stop(fil_space_t *space) noexcept + { space->backup_stop(); } + + /** Delete unnecessary logs that had been created for backup. */ + void delete_logs() noexcept + { + ut_ad(old_size); + for (const lsn_t lsn : logs) + IF_WIN(DeleteFile,unlink)(log_sys.get_archive_path(lsn).c_str()); + } + + /** + Back up a persistent InnoDB data file. + @param target backup target directory + @param node InnoDB data file + @param start first page number + @return error code (non-positive) + @retval 0 on success + */ + static int backup(IF_WIN(const char *,int) target, + fil_node_t *node, uint32_t start) noexcept + { + for (bool tried_mkdir{false};;) + { +#ifdef _WIN32 + std::string path{target}; + path.push_back('/'); + path.append(node->name); + HANDLE f= CreateFile(path.c_str(), GENERIC_WRITE, 0, + my_win_file_secattr(), CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (f == INVALID_HANDLE_VALUE) + { + unsigned long err= GetLastError(); + if (err == ERROR_PATH_NOT_FOUND && !tried_mkdir && + node->space->id && !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + path.erase(path.rfind('/')); + if (CreateDirectory(path.c_str(), + my_dir_security_attributes.lpSecurityDescriptor + ? &my_dir_security_attributes : nullptr) || + (err= GetLastError()) == ERROR_ALREADY_EXISTS) + continue; + } + + my_osmaperr(err); + goto fail; + } +#else + int f; +# ifdef __APPLE__ + /* aio::synchronous() in another thread may concurrently invoke + pwrite(2) on node->handle. We assume that both pwrite(2) and + fclonefileat(2) are atomic with respect to each other. Should + this assumption be invalid, some data files in the backup may be + corrupted. This corruption can be fixed by either removing this + special handling, or by implementing file-level locking. */ + f= fclonefileat(node->handle, target, node->name, 0); + if (!f) + break; + switch (errno) { + case ENOENT: + goto try_mkdir; + case ENOTSUP: + break; + default: + goto fail; + } +# endif + f= openat(target, node->name, + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (f < 0) + { + if (errno == ENOENT) + { +# ifdef __APPLE__ + try_mkdir: +# endif + if (!tried_mkdir && node->space->id && + !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + const char *sep= strchr(node->name, '/'); + ut_ad(sep); + sep= strchr(sep + 1, '/'); + ut_ad(sep); + std::string dir{node->name, size_t(sep - node->name)}; + if (!mkdirat(target, dir.c_str(), 0777) || errno == EEXIST) + continue; + } + } + goto fail; + } +#endif + int err{0}; + for (const uint32_t file_size{node->size}, + page_size{node->space->physical_size()};;) + { + const uint32_t end{start + fil_space_t::BACKUP_BATCH_SIZE}; + backup_batch_start(node->space, end); + /* TODO: avoid copying freed page ranges */ + err= copy_file(node->handle, f, start * uint64_t{page_size}, + std::min(end, file_size) * uint64_t{page_size}); + backup_batch_stop(node->space); + if (err | ((start= end) >= file_size)) + break; + } + + if (IF_WIN(!CloseHandle(f), close(f)) | err) + goto fail; + break; + } + return 0; + fail: + my_error(ER_CANT_CREATE_FILE, MYF(0), node->name, errno); + return -1; + } + + /** + Stream a persistent InnoDB data file. + @param stream backup target stream + @param node InnoDB data file + @param start first page number + @return error code (non-positive) + @retval 0 on success + */ + static int stream(IF_WIN(HANDLE,int) stream, fil_node_t *node, + uint32_t start) noexcept + { + const uint32_t file_size{node->size}, + page_size{node->space->physical_size()}; + int err= backup_stream_start(stream, node->name, 0644, + uint64_t{file_size} * page_size, + /* TODO: leave holes for freed page ranges */ + nullptr, 0); + while (!err && start < file_size) + { + const uint32_t end{start + fil_space_t::BACKUP_BATCH_SIZE}; + backup_batch_start(node->space, end); + err= backup_stream_append(node->handle, stream, + start * uint64_t{page_size}, + std::min(end, file_size) * + uint64_t{page_size}); + backup_batch_stop(node->space); + start= end; + } + + if (err) + my_error(ER_IO_WRITE_ERROR, MYF(0), errno, strerror(errno), + "BACKUP SERVER"); + return err; + } + +private: + /** + Initialize a checkpoint header buffer pointing to the start of the backup. + @param buf checkpoint buffer + @param c offset of the FILE_CHECKPOINT mini-transaction + */ + static void write_checkpoint_buf(uint64_t *buf, uint64_t c) noexcept + { + ut_ad(c >= log_sys.START_OFFSET); + if (log_sys.is_encrypted()) + log_crypt_write_header(reinterpret_cast(buf), true); + buf[4 * log_sys.is_encrypted()]= my_htobe64(c); + } + + /** Write a checkpoint header pointing to the start of the backup. + @param dst target file + @param buf checkpoint header + @return error code + @retval 0 on success */ + static int write_checkpoint(IF_WIN(HANDLE,int) dst, const void *buf) noexcept + { +#ifdef _WIN32 + using tpool::pwrite; +#endif + for (ssize_t o= 0, count= 64; count;) + { + ssize_t ret= + pwrite(dst, static_cast(buf) + o, count, o); + if (ret <= 0 || ret > count) + return -1; + o+= ret; + count-= ret; + } + return 0; + } + +public: + /** Maximum length of the configuration string */ + static constexpr size_t CONFIG_SIZE= + sizeof "[server]\n# checkpoint=" + + sizeof "innodb_log_recovery_start=" + + sizeof "innodb_log_recovery_target=\n" + 45 * 3; + + /** Write the configuration parameters for restoring the backup + @param config buffer for configuration string + @param ctx backup context + @return size of the configuration string */ + static size_t write_config_buf(char *config, const context &ctx) + noexcept + { + ut_ad(ctx.last_lsn != LSN_MAX); + return size_t(snprintf(config, CONFIG_SIZE, + "[server]\n# checkpoint=" LSN_PF "\n" + "innodb_log_recovery_start=" LSN_PF "\n" + "innodb_log_recovery_target=" LSN_PF "\n", + ctx.checkpoint, ctx.checkpoint_end_lsn, + ctx.last_lsn)); + } + + /** Write the configuration parameters for restoring the backup + @param target backup target + @param sink backup worker context + @param ctx backup context + @return error code (non-positive) + @retval 0 on success */ + static int write_config(const backup_target &target, + const backup_sink &sink) noexcept + { + char config[CONFIG_SIZE]; + const size_t size + {write_config_buf(config, *static_cast(sink.ha_data))}; + return sink.stream == sink.NO_STREAM + ? backup_config_append(IF_WIN(target.path, target.fd), config, size) + : backup_stream_config(sink.stream, config, size); + } + + /** + Hard-link (copy) or rename (move) or stream an archive log file. + @param lsn The first LSN in the file + @param target backup target + @param sink backup context + @param old lsn < log_sys.get_first_lsn() + @return error code + @retval 0 on success + */ + static int replicate(lsn_t lsn, + const backup_target &target, + const backup_sink &sink, bool old) noexcept + { + ut_ad(log_get_lsn() >= lsn); + const std::string p{log_sys.get_archive_path(lsn)}; + const char *const path= p.c_str(), *basename= strrchr(path, '/'); + if (!basename) + basename= path; + else + basename++; + context &ctx{*static_cast(sink.ha_data)}; + const bool move{old && !ctx.archived}; + uint64_t cp_buf[8]{}; +#ifdef _WIN32 + ut_ad(!target.path == (sink.stream != sink.NO_STREAM)); + std::string b; + const char *destname= nullptr; + if (!target.path) + goto send_file; + b= target.path; + b.push_back('/'); + b.append(basename); + destname= b.c_str(); + unsigned long err; + if (move) + { + if (!MoveFileEx(path, destname, MOVEFILE_COPY_ALLOWED)) + { + fail: + err= GetLastError(); + got_err: + my_osmaperr(err); + if (target.path) + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), path, basename, + errno); + else + my_error(ER_IO_WRITE_ERROR, MYF(ME_ERROR_LOG), + errno, strerror(errno), "BACKUP SERVER"); + return -1; + } + + if (lsn < ctx.checkpoint) + { + if (!SetFileAttributes(destname, FILE_ATTRIBUTE_NORMAL)) + goto fail; + HANDLE dh= CreateFile(destname, GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (dh == INVALID_HANDLE_VALUE) + goto fail; + if (os_file_set_sparse_win32(dh)) + std::ignore= + os_file_punch_hole(dh, 0, log_sys.START_OFFSET + + ((ctx.checkpoint - lsn) & ~4095ULL)); + write_checkpoint_buf(cp_buf, ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + int fail= write_checkpoint(dh, cp_buf); + std::ignore= CloseHandle(dh); + if (fail) + goto fail; + } + return 0; + } + else if (CreateHardLink(destname, path, nullptr)) + { + ctx.note_hardlink(lsn); + return 0; + } + + if ((err= GetLastError()) != ERROR_NOT_SAME_DEVICE) + goto got_err; + /* Hard-linking failed. Try copying with the final name. */ + if (target.path) + { + b= target.path; + b.push_back('/'); + b.append(basename); + destname= b.c_str(); + + if (lsn >= ctx.checkpoint && lsn < ctx.max_first_lsn) + { + /* Copy a middle log file entirely. */ + if (CopyFileEx(path, basename, nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING)) + return 0; + goto fail; + } + } + + send_file: + HANDLE src; + for (;;) + { + src= CreateFile(path, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (src != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + goto fail; + } + HANDLE dst{sink.stream}; + if (dst == INVALID_HANDLE_VALUE) + { + dst= CreateFile(destname, GENERIC_WRITE, 0, my_win_file_secattr(), + CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); + if (dst == INVALID_HANDLE_VALUE) + { + std::ignore= CloseHandle(src); + goto fail; + } + } +#else + if (sink.stream != sink.NO_STREAM); + else if (move + ? !renameat(AT_FDCWD, path, target.fd, basename) + : !linkat(AT_FDCWD, path, target.fd, basename, AT_SYMLINK_FOLLOW)) + { + if (!move) + ctx.note_hardlink(lsn); +# ifdef __linux__ + else if (lsn != ctx.first_lsn); + else if (off_t garbage= (ctx.checkpoint - lsn) & ~4095ULL) + /* Best effort to punch a hole to free up some garbage in + the first file. We do not care about failures. */ + if (!fchmodat(target.fd, basename, 0644, 0)) + { + int dst= openat(target.fd, basename, O_RDWR); + if (dst >= 0) + fallocate(dst, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + log_sys.START_OFFSET, garbage); + close(dst); + std::ignore= fchmodat(target.fd, basename, 0444, 0); + } +# endif + return 0; + } + else if (errno != EXDEV) + { + fail: + if (sink.stream == sink.NO_STREAM) + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), path, basename, errno); + else + my_error(ER_IO_WRITE_ERROR, MYF(ME_ERROR_LOG), errno, strerror(errno), + "BACKUP SERVER"); + return -1; + } + + const int src{open(path, O_RDONLY)}; + if (src < 0) + goto fail; + if (move && unlink(path)) + { + close_and_fail: + std::ignore= close(src); + goto fail; + } + int dst{sink.stream}; + if (dst < 0) + { + dst= openat(target.fd, basename, + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (dst < 0) + goto close_and_fail; + } + int err; +#endif + backup_chunk chunks[3], *chunk{chunks}; + *chunk++= {log_sys.START_OFFSET, ctx.last_lsn - lsn}; + if (lsn < ctx.checkpoint) + { + /* Copy the necessary part of the first log file. */ + ut_ad(lsn == ctx.first_lsn); + write_checkpoint_buf(cp_buf, ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + chunk[-1]= {0, 512}; + const lsn_t end= + std::min(ctx.last_lsn, lsn + ctx.first_size - log_sys.START_OFFSET); + *chunk++= + {log_sys.START_OFFSET + ctx.checkpoint - lsn, end - ctx.checkpoint}; + chunk->offset= end - lsn; + goto pad_size; + } + else if (lsn < ctx.max_first_lsn) + { + /* Copy a middle log file entirely. */ +#ifdef _WIN32 + ut_ad(dst == sink.stream); + chunk->offset= os_file_get_size(src); +#else + if (dst != sink.stream) + { + err= copy_entire_file(src, dst); + goto close_dst; + } + chunk->offset= uint64_t(lseek(src, 0, SEEK_END)); +#endif + /* Omit the checkpoint header from the stream. */ + chunk[-1].length= chunk->offset - log_sys.START_OFFSET; + goto stream_file; + } + else + { + ut_ad(ctx.max_first_lsn == lsn); + ut_ad(ctx.last_lsn > lsn); + ut_ad(ctx.last_lsn != LSN_MAX); + ut_ad(chunk[-1].length == ctx.last_lsn - lsn); + chunk->offset= chunk[-1].length; + pad_size: + /* Set the logical size of the file. */ + chunk->offset= + std::max(log_sys.FILE_SIZE_MIN, + (chunk->offset + (log_sys.START_OFFSET + 4095)) & + ~4095ULL); + } + + if (dst == sink.stream) + { + stream_file: + chunk++->length= 0; + const backup_chunk &end{chunk[-2]}; + ut_ad(chunk - chunks == 2 || chunk - chunks == 3); + const size_t cp_size{(size_t(chunk - chunks) & 1) << 9}; + err= backup_stream_start(dst, basename, + 0444 | int{lsn == ctx.max_first_lsn} << 7, + end.length + cp_size, chunks, chunk - chunks); + if (!err && cp_size) + err= backup_stream_write(dst, cp_buf, sizeof cp_buf) || + backup_stream_write(dst, field_ref_zero, cp_size - sizeof cp_buf); + if (!err) + { + err= backup_stream_append_async(src, dst, end.offset, + end.offset + end.length); + if (err); + else if (size_t pad= size_t(end.length) & 511) + err= backup_stream_write(dst, field_ref_zero, 512 - pad); + } + } + else + { + /* First, extend the file to a valid size. */ +#ifdef _WIN32 + LARGE_INTEGER li; + li.QuadPart= chunk->offset; + err= !SetFilePointerEx(dst, li, nullptr, FILE_BEGIN) || + !SetEndOfFile(dst) || +#else + err= ftruncate(dst, chunk->offset) || +#endif + copy_file(src, dst, chunk[-1].offset, chunk[-1].offset + + chunk[-1].length) || + (lsn < ctx.checkpoint && write_checkpoint(dst, cp_buf)); +#ifdef _WIN32 + err|= !CloseHandle(dst); +#else + close_dst: + err|= close(dst); +#endif + } + + if (err | IF_WIN(!CloseHandle(src), close(src))) + goto fail; + + return 0; + } +}; + +/** The backup context; protected by log_sys.latch */ +static InnoDB_backup innodb_backup; +} + +bool log_t::backup_start(uint64_t *old_size, THD *thd) noexcept +{ + ut_ad(latch_have_wr()); + ut_ad(!backup); + backup= true; + *old_size= 0; + if (archive) + return false; + const uint64_t old_file_size{file_size}; + latch.wr_unlock(); + const bool fail{set_archive(true, thd, true)}; + latch.wr_lock(); + if (!fail) + { + *old_size= old_file_size; + return false; + } + ut_ad(backup); + backup= false; + const uint64_t new_file_size{file_size}; + latch.wr_unlock(); + if (old_file_size != new_file_size && old_file_size && + resize_start(old_file_size, thd) == RESIZE_STARTED) + resize_finish(thd); + latch.wr_lock(); + return true; +} + +void log_t::backup_stop(uint64_t old_size, THD *thd) noexcept +{ + ut_ad(latch_have_wr()); + /* We will be invoked with old_size=0 after a failed backup_start(), + or if innodb_log_archive=ON held during a successful backup_start(). */ + ut_ad(!old_size || !resize_in_progress()); + ut_ad(!old_size || backup); + backup= false; + const uint64_t new_size{file_size}; + latch.wr_unlock(); + if (old_size && old_size != new_size && + resize_start(old_size, thd) == RESIZE_STARTED) + resize_finish(thd); +} + +void *innodb_backup_start(THD *thd, const backup_target *, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + case BACKUP_PHASE_START: + return innodb_backup.init(thd); + case BACKUP_PHASE_NO_COMMIT: + innodb_backup.commit(); + /* fall through */ + default: + return sink->ha_data; + } +} + +int innodb_backup_step(THD *, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + case BACKUP_PHASE_START: + case BACKUP_PHASE_NO_COMMIT: + case BACKUP_PHASE_FINISH: + return innodb_backup.step(*target, phase, *sink); + default: + return 0; + } +} + +int innodb_backup_end(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + default: + return 0; + case BACKUP_PHASE_FINISH: + return innodb_backup.fini(*target, *sink); + case BACKUP_PHASE_NO_COMMIT: + case BACKUP_PHASE_ABORT: + return innodb_backup.end(thd, phase, *sink); + } +} + +void innodb_backup_checkpoint() noexcept +{ + innodb_backup.checkpoint_complete(); +} diff --git a/storage/innobase/handler/backup_innodb.h b/storage/innobase/handler/backup_innodb.h new file mode 100644 index 0000000000000..e9393038e5a21 --- /dev/null +++ b/storage/innobase/handler/backup_innodb.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/** + Start of a BACKUP SERVER phase, + when no innodb_backup_step() or innodb_backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to sink, or nullptr + @retval -1 on failure +*/ +void *innodb_backup_start(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) + noexcept; + +/** + Process a file that was collected in innodb_backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @return number of files remaining, or negative on error + @retval 0 on completion +*/ +int innodb_backup_step(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept; + +/** + Finish a phase, once all calls for the current phase are completed. + @param thd current sesssion + @param target backup target + @param phase last phase on which backup_start() was successfully invoked, + or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success +*/ +int innodb_backup_end(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept; + +/** + Complete the first checkpoint in a new archive log file. +*/ +void innodb_backup_checkpoint() noexcept; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 92d6a2ddc7968..cc759f884bd6b 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -152,6 +152,7 @@ MDL_ticket *get_mdl_ticket(TABLE *table) noexcept; #include "ha_innodb.h" #include "i_s.h" +#include "backup_innodb.h" #include #include @@ -4195,6 +4196,9 @@ static int innodb_init(void* p) = innodb_prepare_commit_versioned; innobase_hton->update_optimizer_costs= innobase_update_optimizer_costs; + innobase_hton->backup_start = innodb_backup_start; + innobase_hton->backup_step = innodb_backup_step; + innobase_hton->backup_end = innodb_backup_end; innobase_hton->binlog_init= innodb_binlog_init; innobase_hton->set_binlog_max_size= ibb_set_max_size; innobase_hton->binlog_write_direct_ordered= @@ -18867,39 +18871,7 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, ib_senderrf(thd, IB_LOG_LEVEL_ERROR, ER_CANT_CREATE_HANDLER_FILE); break; case log_t::RESIZE_STARTED: - for (timespec abstime;;) - { - if (thd_kill_level(thd)) - { - log_sys.resize_abort(thd); - break; - } - - set_timespec(abstime, 5); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - lsn_t resizing= log_sys.resize_in_progress(); - if (resizing > buf_pool.get_oldest_modification(0)) - { - buf_pool.page_cleaner_wakeup(true); - my_cond_timedwait(&buf_pool.done_flush_list, - &buf_pool.flush_list_mutex.m_mutex, &abstime); - resizing= log_sys.resize_in_progress(); - } - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (!resizing || !log_sys.resize_running(thd)) - break; - log_sys.latch.wr_lock(); - while (resizing > log_sys.get_lsn()) - { - ut_ad(!log_sys.is_mmap()); - /* The server is almost idle. Write dummy FILE_CHECKPOINT records - to ensure that the log resizing will complete. */ - mtr_t mtr{nullptr}; - mtr.start(); - mtr.commit_files(log_sys.last_checkpoint_lsn); - } - log_sys.latch.wr_unlock(); - } + log_sys.resize_finish(thd); } } mysql_mutex_lock(&LOCK_global_system_variables); @@ -19756,7 +19728,9 @@ static void innodb_log_archive_update(THD *thd, st_mysql_sys_var*, { /* MDEV-36828 TODO: On failure, report which other setting conflicted with the request */ + mysql_mutex_unlock(&LOCK_global_system_variables); log_sys.set_archive(*static_cast(save), thd); + mysql_mutex_lock(&LOCK_global_system_variables); } static MYSQL_SYSVAR_BOOL(log_archive, log_sys.archive, @@ -19769,10 +19743,20 @@ static MYSQL_SYSVAR_UINT64_T(log_archive_start, innodb_log_archive_start, "initial value of innodb_lsn_archived; 0=auto-detect", nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); +static void innodb_log_recovery_start_update(THD *, st_mysql_sys_var*, + void *, const void *save) noexcept +{ + const lsn_t lsn{*static_cast(save)}; + recv_sys.recovery_start= lsn; + if (lsn && log_sys.archive) + log_sys.archived_checkpoint= lsn; +} + static MYSQL_SYSVAR_UINT64_T(log_recovery_start, recv_sys.recovery_start, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + PLUGIN_VAR_RQCMDARG, "LSN to start recovery from (0=automatic)", - nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); + nullptr, innodb_log_recovery_start_update, + 0, 0, std::numeric_limits::max(), 0); static MYSQL_SYSVAR_UINT64_T(log_recovery_target, recv_sys.rpo, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 67fa1bfd026f0..b5476a0c0ed89 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -408,6 +408,13 @@ struct fil_space_t final /** Whether any corruption of this tablespace has been reported */ mutable std::atomic_flag is_corrupted= ATOMIC_FLAG_INIT; + /** BACKUP SERVER flag in write_or_backup */ + static constexpr uint8_t BACKUP{128}; + /** whether there is a pending write or backup */ + std::atomic write_or_backup{0}; + /** first page number that is not being backed up */ + std::atomic backup_end{0}; + public: /** mutex to protect freed_ranges and last_freed_lsn */ std::mutex freed_range_mutex; @@ -1058,6 +1065,46 @@ struct fil_space_t final VALIDATE_IMPORT }; + /** Note that writes are being submitted to the tablespace. + @return whether a backup is pending */ + bool writing_start() noexcept + { + uint8_t wb{write_or_backup.fetch_add(1, std::memory_order_acq_rel)}; + ut_ad(~wb & (BACKUP - 1)); + return wb & BACKUP; + } + + /** Note that we there are no more pending writes to the tablespace. */ + void writing_stop() noexcept + { + ut_d(uint8_t wb=) write_or_backup.fetch_sub(1, std::memory_order_release); + ut_ad(wb & ~BACKUP); + } + + /** Note that we backing up some pages of the underlying files. + @param last_page the last page that is being backed up */ + bool backup_start(uint32_t last_page) noexcept + { + backup_end.store(last_page, std::memory_order_relaxed); + uint8_t wb{write_or_backup.fetch_add(BACKUP, std::memory_order_acq_rel)}; + ut_ad(!(wb & BACKUP)); + return wb & ~BACKUP; + } + /** Note that we are not currently backing up the underlying files. */ + void backup_stop() noexcept + { + backup_end.store(0, std::memory_order_relaxed); + ut_d(uint8_t wb=) + write_or_backup.fetch_sub(BACKUP, std::memory_order_release); + ut_ad(wb & BACKUP); + } + /** @return the first page number that is not being backed up */ + uint32_t backup_page_end() const noexcept + { return backup_end.load(std::memory_order_relaxed); } + + /** The size of a backup copy_file() batch in pages */ + static constexpr uint32_t BACKUP_BATCH_SIZE{64}; + /** Update the data structures on write completion */ void complete_write() noexcept; diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 44a827dbf636d..6d07fa25e013d 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -221,6 +221,8 @@ struct log_t /** whether !archive log records may have been written with get_sequence_bit()==0 */ bool circular_recovery_from_sequence_bit_0:1; + /** whether we are between backup_start() and backup_stop() */ + bool backup:1; public: /** the default value of log_mmap */ static constexpr bool log_mmap_default= @@ -288,6 +290,8 @@ struct log_t Atomic_relaxed last_checkpoint_lsn; /** The log writer (protected by latch.wr_lock()) */ lsn_t (*writer)() noexcept; + /** the earliest available checkpoint; protected by latch.wr_lock() */ + lsn_t archived_checkpoint; /** end_lsn of the first available checkpoint, or 0; protected by latch.wr_lock() */ lsn_t archived_lsn; @@ -369,11 +373,24 @@ struct log_t RESIZE_NO_CHANGE, RESIZE_IN_PROGRESS, RESIZE_STARTED, RESIZE_FAILED }; +private: /** Start resizing the log and release the exclusive latch. + @param size requested new file_size + @param thd the current thread identifier + @param backup whether the caller is backup_start() or backup_stop() + @return whether the resizing was started successfully */ + resize_start_status resize_start(uint64_t size, void *thd, bool backup) + noexcept; +public: + /** Start resizing the log. @param size requested new file_size @param thd the current thread identifier @return whether the resizing was started successfully */ - resize_start_status resize_start(os_offset_t size, void *thd) noexcept; + resize_start_status resize_start(uint64_t size, void *thd) noexcept + { return resize_start(size, thd, false); } + + /** Wait for the completion of resize_start() == RESIZE_STARTED */ + void resize_finish(THD *thd) noexcept; /** Abort a resize_start() that we started. @param thd thread identifier that had been passed to resize_start() */ @@ -397,10 +414,37 @@ struct log_t resize_write_low(lsn, end, len, seq); } +private: + /** SET GLOBAL innodb_log_archive, or start/stop BACKUP SERVER + @param archive the new value of innodb_log_archive + @param thd SQL connection + @param backup whether the caller is backup_start() or backup_stop() + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd, bool backup) noexcept; +public: /** SET GLOBAL innodb_log_archive @param archive the new value of innodb_log_archive - @param thd SQL connection */ - void set_archive(my_bool archive, THD *thd) noexcept; + @param thd SQL connection + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd) noexcept + { return set_archive(archive, thd, false); } + + /** Start BACKUP SERVER. + @param old_size the old file_size, or 0 on failure or when + already running innodb_log_archive=ON + @param thd SQL connection + @return whether the operation failed */ + bool backup_start(uint64_t *old_size, THD *thd) noexcept; + /** Stop log archiving in BACKUP SERVER clean-up + @param thd SQL connection + @return whether the operation failed */ + bool backup_stop_archiving(THD *thd) noexcept + { return set_archive(false, thd, true); } + + /** Stop BACKUP SERVER. + @param old_size the value returned by backup_start() + @param thd SQL connection */ + void backup_stop(uint64_t old_size, THD *thd) noexcept; private: /** Replicate a write to the log. @@ -695,6 +739,18 @@ struct log_t /** @return the first LSN of the log file */ lsn_t get_first_lsn() const noexcept { return first_lsn; } + /** + Determine the latest checkpoint. + @param end LSN leading to the FILE_CHECKPOINT record + @return the latest checkpoint LSN + */ + lsn_t get_latest_checkpoint(lsn_t &end) const noexcept + { + ut_ad(latch_have_any()); + end= end_lsn; + return last_checkpoint_lsn; + } + /** Set the recovered checkpoint. @param lsn log sequence number of the checkpoint @param end_lsn LSN passed to write_checkpoint() diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 73d70455da2da..59317f41d8326 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -635,7 +635,7 @@ void log_t::set_buffered(bool buffered) noexcept } #endif - /** Try to enable or disable durable writes (update log_write_through) */ +/** Try to enable or disable durable writes (update log_write_through) */ void log_t::set_write_through(bool write_through) { if (is_mmap() || high_level_read_only || recv_sys.rpo) @@ -764,9 +764,12 @@ void log_t::header_rewrite(my_bool archive) noexcept /** SET GLOBAL innodb_log_archive @param archive the new value of innodb_log_archive -@param thd SQL connection */ -void log_t::set_archive(my_bool archive, THD *thd) noexcept +@param thd SQL connection +@param backup whether the caller is backup_start() or backup_stop() +@return whether the operation failed */ +bool log_t::set_archive(my_bool archive, THD *thd, bool backup) noexcept { + bool fail= false; thd_wait_begin(thd, THD_WAIT_DISKIO); tpool::tpool_wait_begin(); lsn_t wait_lsn; @@ -780,12 +783,20 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept my_printf_error(ER_WRONG_USAGE, "SET GLOBAL innodb_log_file_size is in progress", MYF(0)); + fail: + fail= true; + wait_lsn= 0; break; } if (archive == this->archive) break; - if (thd_kill_level(thd)) - break; + if ((!backup || archive) && thd_kill_level(thd)) + goto fail; + if (!backup && this->backup) + { + my_printf_error(ER_WRONG_USAGE, "BACKUP SERVER is in progress", MYF(0)); + goto fail; + } if (resize_log.is_opened()) { @@ -894,7 +905,7 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept if (!log.is_opened()) { my_error(ER_ERROR_ON_READ, MYF(0), old_name, errno); - break; + goto fail; } } #endif @@ -919,7 +930,7 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept { my_error(ER_ERROR_ON_RENAME, MYF(0), old_name, new_name, my_errno); first_lsn= old_first_lsn; - break; + goto fail; } if (archive) @@ -951,14 +962,16 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept thd_wait_end(thd); if (wait_lsn) mtr_flush_ahead(wait_lsn); + return fail; } -/** Start resizing the log and release the exclusive latch. -@param size requested new file_size -@param thd the current thread identifier +/** Start resizing the log. +@param size requested new file_size +@param thd the current thread identifier +@param backup whether the caller is backup_start() or backup_stop() @return whether the resizing was started successfully */ -log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) - noexcept +log_t::resize_start_status log_t::resize_start(uint64_t size, void *thd, + bool backup) noexcept { ut_ad(size >= 4U << 20); ut_ad(!(size & 4095)); @@ -989,6 +1002,9 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) resize_target= size; } } + else if (!backup && this->backup) + /* backup_start() or backup_stop() is running */ + status= RESIZE_FAILED; else { lsn_t start_lsn; @@ -1090,6 +1106,44 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) return status; } +/** Wait for the completion of resize_start() == RESIZE_STARTED */ +void log_t::resize_finish(THD *thd) noexcept +{ + for (timespec abstime;;) + { + if (thd_kill_level(thd)) + { + resize_abort(thd); + break; + } + + set_timespec(abstime, 5); + mysql_mutex_lock(&buf_pool.flush_list_mutex); + lsn_t resizing= resize_in_progress(); + if (resizing > buf_pool.get_oldest_modification(0)) + { + buf_pool.page_cleaner_wakeup(true); + my_cond_timedwait(&buf_pool.done_flush_list, + &buf_pool.flush_list_mutex.m_mutex, &abstime); + resizing= resize_in_progress(); + } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + if (!resizing || !resize_running(thd)) + break; + latch.wr_lock(); + while (resizing > get_lsn()) + { + ut_ad(!is_mmap()); + /* The server is almost idle. Write dummy FILE_CHECKPOINT records + to ensure that the log resizing will complete. */ + mtr_t mtr{nullptr}; + mtr.start(); + mtr.commit_files(last_checkpoint_lsn); + } + latch.wr_unlock(); + } +} + /** Abort a resize_start() that we started. */ void log_t::resize_abort(void *thd) noexcept { diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 72b62c36d6a15..d1d5c7237599b 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2099,6 +2099,7 @@ dberr_t recv_sys_t::find_checkpoint() memset_aligned<4096>(const_cast(field_ref_zero), 0, 4096); /* Mark the redo log for upgrading. */ lsn= file_checkpoint= log_sys.last_checkpoint_lsn; + log_sys.archived_checkpoint= lsn; log_sys.set_recovered_lsn(lsn); if (rpo && rpo != lsn) { @@ -2177,7 +2178,8 @@ dberr_t recv_sys_t::find_checkpoint() log_sys.set_recovered_checkpoint(checkpoint_lsn, lsn= end_lsn, field == log_t::CHECKPOINT_1); } - if (!log_sys.last_checkpoint_lsn) + log_sys.archived_checkpoint= log_sys.last_checkpoint_lsn; + if (!log_sys.archived_checkpoint) goto got_no_checkpoint; else if (!log_sys.archived_lsn) log_sys.archived_lsn= lsn; diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 6494c5e21b96e..4ef86095f78a3 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2017,7 +2017,15 @@ os_file_create_func( ); DWORD create_flag = OPEN_EXISTING; - DWORD share_mode = read_only + /* BACKUP SERVER may invoke CreateHardLink() on a log file that + may concurrently be written to. This is why we must allow + FILE_SHARE_WRITE. This has the side effect that multiple InnoDB + instances may be concurrently started on the same log file. + However, InnoDB will not write any log before it has successfully + opened data files. As long as the multiple instances are also + opening the same InnoDB data files (such as the system tablespace), + they should fail to start up concurrently. */ + DWORD share_mode = read_only || type == OS_LOG_FILE ? FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE : FILE_SHARE_READ | FILE_SHARE_DELETE; diff --git a/storage/maria/CMakeLists.txt b/storage/maria/CMakeLists.txt index 9d528f980dc5b..fd54198e2641f 100644 --- a/storage/maria/CMakeLists.txt +++ b/storage/maria/CMakeLists.txt @@ -45,6 +45,7 @@ SET(ARIA_SOURCES ma_init.c ma_open.c ma_extra.c ma_info.c ma_rkey.c ha_maria.h maria_def.h ma_recovery_util.c ma_servicethread.c ma_norec.c ma_crypt.c ma_backup.c + ma_backup.cc ma_backup.h ) IF(APPLE) diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 8f5e47daea728..624a7f0a043d3 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -23,6 +23,7 @@ #include #include #include "ha_maria.h" +#include "ma_backup.h" #include "trnman_public.h" #include "trnman.h" @@ -3942,6 +3943,9 @@ static int ha_maria_init(void *p) maria_hton->prepare_for_backup= maria_prepare_for_backup; maria_hton->end_backup= maria_end_backup; maria_hton->update_optimizer_costs= aria_update_optimizer_costs; + maria_hton->backup_start= aria_backup_start; + maria_hton->backup_step= aria_backup_step; + maria_hton->backup_end= aria_backup_end; /* TODO: decide if we support Maria being used for log tables */ maria_hton->flags= (HTON_CAN_RECREATE | HTON_SUPPORT_LOG_TABLES | diff --git a/storage/maria/ma_backup.cc b/storage/maria/ma_backup.cc new file mode 100644 index 0000000000000..b3411bc48f5cd --- /dev/null +++ b/storage/maria/ma_backup.cc @@ -0,0 +1,414 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "maria_def.h" +#include "ma_backup.h" +#include "mysqld_error.h" +#if 1 // tc_purge(), tdc_purge() +# include "sql_class.h" +# include "table_cache.h" +#endif +#include +#include +#include +#include + +/* + Implementation of functions declatred in ma_backup.h: + BACKUP SERVER support for Aria engine +*/ + +namespace +{ + class Source_dir + { + public: + Source_dir(const char* path, myf flags) noexcept + { + dir_info= my_dir(path, flags); + if (!dir_info) + { + my_error(ER_CANT_READ_DIR, MYF(0), path, my_errno); + } + } + ~Source_dir() noexcept + { + my_dirend(dir_info); + } + bool is_error() const noexcept + { + return !dir_info; + } + template + int for_each(Fn fn) const noexcept + { + for (size_t i= 0; i < dir_info->number_of_files; i++) + { + if (fn(dir_info->dir_entry[i]) != 0) + return 1; + } + return 0; + } + + private: + MY_DIR *dir_info {nullptr}; + }; + + + /** Backup state; protected by log_sys.latch */ + class Aria_backup + { + public: + Aria_backup(const backup_target &target, const backup_sink &sink) noexcept + : target(target) +#ifndef _WIN32 + , datadir_fd(open(maria_data_root, O_DIRECTORY)) + { + if (datadir_fd < 0) + { + my_error(ER_CANT_READ_DIR, MYF(0), maria_data_root, errno); + return; + } +#else + { +#endif // _WIN32 + translog_disable_purge(); + } + + bool is_initialized() const noexcept + { +#ifndef _WIN32 + return datadir_fd >= 0; +#else + return true; +#endif // _WIN32 + } + + ~Aria_backup() noexcept + { +#ifndef _WIN32 + if (datadir_fd >= 0) + close(datadir_fd); +#endif // _WIN32 + } + + int end(const backup_sink &sink) noexcept + { + int ret_val= perform_backup(sink); + translog_enable_purge(); + return ret_val; + } + private: + const backup_target ⌖ // FIXME: pass to member functions +#ifndef _WIN32 + const int datadir_fd; +#endif + static constexpr const char zerobuf[511]{}; + static constexpr const char *const data_exts[]= + {"MAD", "MAI", "MYD", "MYI", "frm", "par"}; + using dir_contents = std::vector; + using database_dir = std::pair; + std::vector database_dirs; + std::vector log_files; + bool have_control_file = false; + + int perform_backup(const backup_sink &sink) noexcept + { + return scan_datadir() || copy_databases(sink) || + copy_control_file(sink) || translog_flush(translog_get_horizon()) || + copy_logs(sink); + } + + int scan_datadir() noexcept + { + const char* base_dir = maria_data_root; + Source_dir datadir(base_dir, MYF(MY_WANT_STAT)); + if (datadir.is_error()) + return 1; + datadir.for_each([this](const fileinfo &fi) + { + if (fi.mystat->st_mode & S_IFDIR) + { + if (scan_database_dir(fi.name) != 0) + return 1; + } + else if (!strncmp(fi.name, C_STRING_WITH_LEN("aria_log."))) + log_files.emplace_back(fi.name); + else if (!strcmp(fi.name, "aria_log_control")) + have_control_file = true; + return 0; + }); + return 0; + } + + int scan_database_dir(const char* dir_name) noexcept + { + const char* base_dir = maria_data_root; + std::string dir_path = std::string(base_dir) + "/" + dir_name; + Source_dir db_dir(dir_path.c_str(), MYF(0)); + if (db_dir.is_error()) + return 1; + std::vector files_to_backup; + db_dir.for_each([&files_to_backup](const fileinfo &fi) + { + if (is_db_file(fi.name)) + files_to_backup.emplace_back(fi.name); + return 0; + }); + if (!files_to_backup.empty()) + database_dirs.emplace_back(dir_name, std::move(files_to_backup)); + return 0; + } + + int copy_databases(const backup_sink &sink) noexcept + { + for (const database_dir& dir : database_dirs) + { + if (sink.stream == sink.NO_STREAM && + ensure_target_subdir(dir.first.c_str())) + { + my_error(ER_CANT_CREATE_FILE, MYF(0), dir.first.c_str(), errno); + return 1; + } + if (copy_database(sink, dir) != 0) + return 1; + } + return 0; + } + + /* + Create directory in the target directory if it does not exist. + Return 0 on success, non-0 on failure. Set errno in case of failure + */ + int ensure_target_subdir(const char* name) noexcept + { +#ifdef _WIN32 + std::string dir_path= targetPath() + "/" + name; + if (!CreateDirectory(dir_path.c_str(), nullptr)) + { + DWORD err = GetLastError(); + if (err != ERROR_ALREADY_EXISTS) + { + my_osmaperr(err); + return 1; + } + } +#else + if (mkdirat(target.fd, name, 0777) != 0) + return (errno != EEXIST); +#endif + return 0; + } + + int copy_database(const backup_sink &sink, const database_dir& dir) + noexcept + { + for (const std::string& file : dir.second) + { + std::string file_path= dir.first + "/" + file; + if (copy_file(sink, file_path.c_str()) != 0) + return 1; + } + return 0; + } + + int copy_control_file(const backup_sink &sink) noexcept + { + if (!have_control_file) + return 0; + return copy_file(sink, "aria_log_control"); + } + + int copy_logs(const backup_sink &sink) noexcept + { + for (const std::string& file : log_files) + { + if (copy_file(sink, file.c_str()) != 0) + return 1; + } + return 0; + } + + int copy_file(const backup_sink &sink, const char *path) const noexcept + { +#ifndef _WIN32 + int ret_val = 0; + int src_fd = openat(datadir_fd, path, O_RDONLY); + if (src_fd < 0) + { + my_error(ER_CANT_OPEN_FILE, MYF(0), path, errno); + return 1; + } + int tgt_fd{sink.stream}; + if (tgt_fd == sink.NO_STREAM) + { + tgt_fd= openat(target.fd, path, + O_CREAT | O_EXCL | O_WRONLY, 0666); + if (tgt_fd < 0) + { + my_error(ER_CANT_CREATE_FILE, MYF(0), path, errno); + ret_val = 1; + goto finish; + } + ret_val= copy_entire_file(src_fd, tgt_fd); + if (ret_val | close(tgt_fd)) + { + write_error: + my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); + ret_val = 1; + } + } + else + { + uint64_t end= uint64_t(lseek(src_fd, 0, SEEK_END)); + if (backup_stream_start(tgt_fd, path, 0644, end, nullptr, 0) || + backup_stream_append(src_fd, tgt_fd, 0, end)) + goto write_error; + if (size_t pad= size_t(end) & 511) + if (backup_stream_write(tgt_fd, zerobuf, 512 - pad)) + goto write_error; + } + finish: + close(src_fd); + return ret_val; +#else + std::string src_path= std::string(maria_data_root) + "/" + path; + + if (sink.stream == sink.NO_STREAM) + { + std::string dest_path= targetPath() + "/" + path; + if (!CopyFileEx(src_path.c_str(), dest_path.c_str(), nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_CREATE_FILE, MYF(0), dest_path.c_str(), errno); + return 1; + } + } + else + { + HANDLE src, dst{sink.stream}; + for (;;) + { + src= CreateFile(src_path.c_str(), GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (src != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + + my_osmaperr(GetLastError()); + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), src_path.c_str(), errno); + return -1; + } + + LARGE_INTEGER li; + if (!GetFileSizeEx(src, &li)) + { + write_error: + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); + if (src != INVALID_HANDLE_VALUE) + CloseHandle(src); + return -1; + } + + if (backup_stream_start(dst, path, 0644, li.QuadPart, nullptr, 0) || + backup_stream_append_plain(src, dst, 0, li.QuadPart)) + goto write_error; + + if (size_t pad= size_t(li.LowPart) & 511) + if (backup_stream_write(dst, zerobuf, 512 - pad)) + goto write_error; + if (!CloseHandle(src)) + { + src= INVALID_HANDLE_VALUE; + goto write_error; + } + } + return 0; +#endif + } + + + static bool is_db_file(const char* file_name) noexcept + { + const char *suffix= strrchr(file_name, '.'); + if (!suffix) + return false; + suffix++; + for (const char *ext : data_exts) + if (!strcmp(suffix, ext)) + return true; + return !strcmp(file_name, "db.opt"); + } + +#ifdef _WIN32 + /** @return the target directory path */ + std::string targetPath() const + { + return std::string(target.path); + } +#endif + }; +} + +void *aria_backup_start(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept +{ + if (phase != BACKUP_PHASE_NO_COMMIT) + return sink->ha_data; + assert(!sink->ha_data); + Aria_backup *aria_backup= new Aria_backup(*target, *sink); + if (!aria_backup->is_initialized()) + { + delete aria_backup; + return reinterpret_cast(-1); + } + return aria_backup; +} + +int aria_backup_step(THD*, const backup_target*, backup_phase, + const backup_sink*) noexcept +{ + // FIXME: implement the copying here + return 0; +} + +int aria_backup_end(THD *thd, const backup_target*, backup_phase phase, + const backup_sink *sink) noexcept +{ + Aria_backup *aria_backup= static_cast(sink->ha_data); + switch (phase) { + case BACKUP_PHASE_NO_COMMIT: + assert(aria_backup); +#if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others + tc_purge(); + tdc_purge(true); +#endif + return aria_backup->end(*sink); + case BACKUP_PHASE_FINISH: + delete aria_backup; + /* fall through */ + default: + return 0; + } +} diff --git a/storage/maria/ma_backup.h b/storage/maria/ma_backup.h new file mode 100644 index 0000000000000..6c72bb46eb60f --- /dev/null +++ b/storage/maria/ma_backup.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#pragma once + +/* BACKUP SERVER support for Aria engine. */ + +#include +#include + +/** + Start of a BACKUP SERVER phase, + when no aria_backup_step() or aria_backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to backup_target, or nullptr + @retval -1 on failure +*/ +void *aria_backup_start(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept; + +/** + Process a file that was collected in aria_backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @retval 0 on completion +*/ +int aria_backup_step(THD *thd, const backup_target *target, backup_phase phase, + const backup_sink *sink) noexcept; + +/** + Finish a phase, once all calls for the current phase are completed. + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked, + or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success +*/ +int aria_backup_end(THD *thd, const backup_target *target, backup_phase phase, + const backup_sink *sink) noexcept; From 9ac620ad9c3d873a5352f4f5e318a055daa323c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 25 Jun 2026 19:13:03 +0300 Subject: [PATCH 02/14] fixup! 4769a43b51c58b0fd02b1164a6f80ab916d21903 --- sql/sql_backup.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc index 3351f8fdc3dcd..fddb9f2f175da 100644 --- a/sql/sql_backup.cc +++ b/sql/sql_backup.cc @@ -77,7 +77,7 @@ template static ssize_t mmap_copy(int in_fd, int out_fd, uint64_t o, uint64_t end) { # if SIZEOF_SIZE_T < 8 - if (end != ssize_t(end)) + if (end != size_t(end)) return 1; # endif const size_t count= size_t(end - o); From 301f196d8d87325bfa6e791d816d1fde213328d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 26 Jun 2026 09:22:31 +0300 Subject: [PATCH 03/14] Remove an unnecessary suppression --- mysql-test/suite/backup/backup_stream.test | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mysql-test/suite/backup/backup_stream.test b/mysql-test/suite/backup/backup_stream.test index 649a5557b5b55..0ad21d49a663c 100644 --- a/mysql-test/suite/backup/backup_stream.test +++ b/mysql-test/suite/backup/backup_stream.test @@ -2,10 +2,6 @@ --source include/have_innodb.inc --source include/maybe_debug.inc ---disable_query_log -call mtr.add_suppression("mariadbd.*: IO Write error:"); ---enable_query_log - CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) ENGINE=INNODB; INSERT INTO t SET a=1; From fffd4b65b39466cf66d000261ed0cec9323ac80a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 26 Jun 2026 11:55:47 +0300 Subject: [PATCH 04/14] fixup! 4769a43b51c58b0fd02b1164a6f80ab916d21903 Eliminate the buggy Source_dir that broke error propagation. Also, do not misidentify Linux block devices or sockets as directories. --- storage/maria/ma_backup.cc | 83 +++++++++++++------------------------- 1 file changed, 29 insertions(+), 54 deletions(-) diff --git a/storage/maria/ma_backup.cc b/storage/maria/ma_backup.cc index b3411bc48f5cd..d681c974e8f48 100644 --- a/storage/maria/ma_backup.cc +++ b/storage/maria/ma_backup.cc @@ -24,6 +24,7 @@ #include #include #include +#include "span.h" /* Implementation of functions declatred in ma_backup.h: @@ -32,41 +33,6 @@ namespace { - class Source_dir - { - public: - Source_dir(const char* path, myf flags) noexcept - { - dir_info= my_dir(path, flags); - if (!dir_info) - { - my_error(ER_CANT_READ_DIR, MYF(0), path, my_errno); - } - } - ~Source_dir() noexcept - { - my_dirend(dir_info); - } - bool is_error() const noexcept - { - return !dir_info; - } - template - int for_each(Fn fn) const noexcept - { - for (size_t i= 0; i < dir_info->number_of_files; i++) - { - if (fn(dir_info->dir_entry[i]) != 0) - return 1; - } - return 0; - } - - private: - MY_DIR *dir_info {nullptr}; - }; - - /** Backup state; protected by log_sys.latch */ class Aria_backup { @@ -131,44 +97,53 @@ namespace copy_logs(sink); } + ATTRIBUTE_COLD ATTRIBUTE_NOINLINE + static int dir_error(const char *name) noexcept + { + my_error(ER_CANT_READ_DIR, MYF(0), name, my_errno); + return 1; + } + int scan_datadir() noexcept { - const char* base_dir = maria_data_root; - Source_dir datadir(base_dir, MYF(MY_WANT_STAT)); - if (datadir.is_error()) - return 1; - datadir.for_each([this](const fileinfo &fi) - { - if (fi.mystat->st_mode & S_IFDIR) + const char *base_dir= maria_data_root; + MY_DIR *dir_info= my_dir(base_dir, MYF(MY_WANT_STAT)); + if (!dir_info) + return dir_error(base_dir); + int fail= 0; + for (const fileinfo &fi : + st_::span{dir_info->dir_entry, + dir_info->number_of_files}) + if ((fi.mystat->st_mode & S_IFMT) == S_IFDIR) { - if (scan_database_dir(fi.name) != 0) - return 1; + if ((fail= scan_database_dir(fi.name)) != 0) + goto func_exit; } else if (!strncmp(fi.name, C_STRING_WITH_LEN("aria_log."))) log_files.emplace_back(fi.name); else if (!strcmp(fi.name, "aria_log_control")) have_control_file = true; - return 0; - }); - return 0; + func_exit: + my_dirend(dir_info); + return fail; } int scan_database_dir(const char* dir_name) noexcept { const char* base_dir = maria_data_root; std::string dir_path = std::string(base_dir) + "/" + dir_name; - Source_dir db_dir(dir_path.c_str(), MYF(0)); - if (db_dir.is_error()) - return 1; + MY_DIR *dir_info= my_dir(dir_path.c_str(), MYF(MY_WANT_STAT)); + if (!dir_info) + return dir_error(dir_path.c_str()); std::vector files_to_backup; - db_dir.for_each([&files_to_backup](const fileinfo &fi) - { + for (const fileinfo &fi : + st_::span{dir_info->dir_entry, + dir_info->number_of_files}) if (is_db_file(fi.name)) files_to_backup.emplace_back(fi.name); - return 0; - }); if (!files_to_backup.empty()) database_dirs.emplace_back(dir_name, std::move(files_to_backup)); + my_dirend(dir_info); return 0; } From 18490c8549acc7da75e4cbba4202b363e475921f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 26 Jun 2026 13:48:11 +0300 Subject: [PATCH 05/14] fixup! 4769a43b51c58b0fd02b1164a6f80ab916d21903 InnoDB_backup::context::de_hardlink(): Avoid copying garbage at the start of the first file. --- storage/innobase/handler/backup_innodb.cc | 66 ++++++++++------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index 558c9b3f36f4c..891c831f76e3a 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -117,40 +117,12 @@ class InnoDB_backup CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); if (d == INVALID_HANDLE_VALUE) { - fail: + error_return: my_osmaperr(GetLastError()); std::ignore= CloseHandle(s); my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), s_, d_, errno); return 1; } - const uint64_t payload_end{log_sys.START_OFFSET + last_lsn - hl}; - /* First, extend the file to a valid size. */ - int err; - { - LARGE_INTEGER li; - li.QuadPart= std::max(log_sys.FILE_SIZE_MIN, - (payload_end + 4095) & ~4095ULL); - err= !SetFilePointerEx(d, li, nullptr, FILE_BEGIN) || !SetEndOfFile(d); - } - if (!err && !(err= copy_file(s, d, log_sys.START_OFFSET, payload_end)) && - hl == first_lsn) - { - uint64_t cp_buf[8]{}; - write_checkpoint_buf(cp_buf, - checkpoint_end_lsn - hl + log_sys.START_OFFSET); - err= write_checkpoint(d, cp_buf); - } - if (!CloseHandle(d) | err) - goto fail; - - std::ignore= CloseHandle(s); - - if (!DeleteFile(s_)) - { - my_osmaperr(GetLastError()); - my_error(ER_CANT_DELETE_FILE, MYF(ME_ERROR_LOG), s_, errno); - return 1; - } #else std::string dst; log_sys.append_archive_name(dst, hl); @@ -173,22 +145,42 @@ class InnoDB_backup d= openat(target.fd, d_, O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); if (d < 0) goto error_return; - const uint64_t payload_end{log_sys.START_OFFSET + last_lsn - hl}; +#endif + const uint64_t end{log_sys.START_OFFSET + last_lsn - hl}; /* First, extend the file to a valid size. */ - int fail= +#ifdef _WIN32 + int f; + { + LARGE_INTEGER li; + li.QuadPart= std::max(log_sys.FILE_SIZE_MIN, + (end + 4095) & ~4095ULL); + f= !SetFilePointerEx(d, li, nullptr, FILE_BEGIN) || !SetEndOfFile(d); + } +#else + int f= ftruncate(d, std::max(log_sys.FILE_SIZE_MIN, - (payload_end + 4095) & ~4095LL)) || - copy_file(s, d, log_sys.START_OFFSET, payload_end); - if (!fail && hl == first_lsn) + (end + 4095) & ~4095LL)); +#endif + if (!f && + !(f= copy_file(s, d, log_sys.START_OFFSET + + (hl == first_lsn) * (checkpoint - hl), end)) && + hl == first_lsn) { uint64_t cp_buf[8]{}; write_checkpoint_buf(cp_buf, checkpoint_end_lsn - hl + log_sys.START_OFFSET); - fail= write_checkpoint(d, cp_buf); + f= write_checkpoint(d, cp_buf); } - if (close(d) | fail) + if (IF_WIN(!CloseHandle(d), close(d)) | f) goto error_return; - std::ignore= close(s); + std::ignore= IF_WIN(CloseHandle(s), close(s)); +#ifdef _WIN32 + if (!DeleteFile(s_)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_DELETE_FILE, MYF(ME_ERROR_LOG), s_, errno); + return 1; + } #endif return 0; } From a7570c419353b470892ecf7909a1bc109f4058e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 26 Jun 2026 14:30:59 +0300 Subject: [PATCH 06/14] squash! 4769a43b51c58b0fd02b1164a6f80ab916d21903 fil_system.have_all_spaces: A flag that indicates whether all tablespace metadata is present in fil_system. --- storage/innobase/dict/dict0load.cc | 5 +++++ storage/innobase/handler/backup_innodb.cc | 6 ++++++ storage/innobase/include/fil0fil.h | 2 ++ 3 files changed, 13 insertions(+) diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index d8976dec66974..fe221f72f0873 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -889,6 +889,10 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) dict_sys.lock(SRW_LOCK_CALL); + if (fil_system.have_all_spaces) { + goto done; + } + if (!spaces && !upgrade && !encryption_key_id_exists(FIL_DEFAULT_ENCRYPTION_KEY)) { max_space_id = dict_find_max_space_id(&pcur, &mtr); @@ -985,6 +989,7 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) ut_free(filepath); } + fil_system.have_all_spaces = true; done: mtr.commit(); diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index 891c831f76e3a..e4674412c0b29 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -20,6 +20,7 @@ #include "trx0trx.h" #include "buf0flu.h" #include "log0crypt.h" +#include "dict0load.h" #include #ifdef __linux__ # include @@ -260,6 +261,11 @@ class InnoDB_backup start, start_end, !old_size, false, 0 }; + if (!fil_system.have_all_spaces) { + dict_load_tablespaces(nullptr, true); + ut_ad(fil_system.have_all_spaces); + } + /* Collect all tablespaces that have been created before our start checkpoint. Newer tablespaces will be recovered by the innodb_log_archive=ON recovery. diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index b5476a0c0ed89..4111d4d399d8d 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1510,6 +1510,8 @@ struct fil_system_t my_bool buffered; /** whether fdatasync() is needed on data files */ Atomic_relaxed need_unflushed_spaces; + /** whether dict_load_tablespaces(nullptr, true) is unnecessary */ + Atomic_relaxed have_all_spaces; /** Try to enable or disable write-through of data files */ void set_write_through(bool write_through); From 554a791a751a2dcbb1edd47d317f0fdd03c24b01 Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Thu, 28 May 2026 14:31:32 +0200 Subject: [PATCH 07/14] Implement table and file copy using backup_step --- storage/maria/ma_backup.cc | 489 +++++++++++++++++++++++++++++++------ 1 file changed, 411 insertions(+), 78 deletions(-) diff --git a/storage/maria/ma_backup.cc b/storage/maria/ma_backup.cc index d681c974e8f48..6a34bc7250984 100644 --- a/storage/maria/ma_backup.cc +++ b/storage/maria/ma_backup.cc @@ -20,11 +20,17 @@ # include "sql_class.h" # include "table_cache.h" #endif +#include +#include +#include #include #include #include #include #include "span.h" +#include +#include +#include /* Implementation of functions declatred in ma_backup.h: @@ -33,11 +39,62 @@ namespace { - /** Backup state; protected by log_sys.latch */ + /* Utility class to implement the "backup step" interface when + processing several lists. It implements the logic where an item + is processed (copied) from the first list which has available + items, and a "remaining" counter accumulates the number of + items remaining to be processed on all lists, regardless of + whether an item from that list was processed or not. */ + class Copy_from_list + { + int m_remaining {0}; + bool m_copy_done; + public: + Copy_from_list(bool copy_done= false) noexcept + : m_copy_done(copy_done) + { + } + + bool copy_done() const noexcept + { + return m_copy_done; + } + + int remaining() const noexcept + { + return m_remaining; + } + + template + bool operator()(const T &list, std::atomic &copied, + Fn copy_action) noexcept + { + if(!m_copy_done) + { + size_t idx= copied.fetch_add(1, std::memory_order_relaxed); + if (idx < list.size()) + { + if (copy_action(list[idx]) != 0) + return 1; + m_copy_done= true; + m_remaining+= list.size() - idx - 1U; + } + } + else + { + size_t current_copied= copied.load(std::memory_order_relaxed); + if (current_copied < list.size()) + m_remaining+= list.size() - current_copied; + } + return 0; + } + }; + + class Aria_backup { public: - Aria_backup(const backup_target &target, const backup_sink &sink) noexcept + Aria_backup(const backup_target &target) noexcept : target(target) #ifndef _WIN32 , datadir_fd(open(maria_data_root, O_DIRECTORY)) @@ -70,11 +127,90 @@ namespace #endif // _WIN32 } - int end(const backup_sink &sink) noexcept + bool start_copy_dml_safe(const backup_sink &sink) noexcept + { + if (scan_dbdirs()) + return true; + flatten_table_lists(); + if (sink.stream == sink.NO_STREAM) + return ensure_target_dirs(); + return false; + } + + bool start_copy_unsafe() noexcept + { + if (scan_logs()) + return true; + return false; + } + + /* Copy an Aria table that is safe to be copied while concurrent DML + is in progress. */ + int dml_safe_copy_step(const backup_sink &sink) noexcept + { + Copy_from_list copy_from_list; + auto copy_table_action= [this, sink](const table_ref &table) noexcept + { + return copy_table(sink, table); + }; + if (copy_from_list(dml_safe_table_list, dml_safe_tables_copied, + copy_table_action) != 0) + return -1; + if (copy_from_list(unsafe_tables_list, unsafe_tables_copied, + copy_table_action) != 0) + return -1; + if (copy_from_list(misc_files, misc_files_copied, + [this, sink](const std::string &path) noexcept + { + return copy_file(sink, path); + }) != 0) + return -1; + return copy_from_list.remaining(); + } + + /* Copy an entity that is not safe to copy if there are concurrent + writes to it. One entity is copied, of the first category that has + any remaning entities to be copied. Returns the total number of + entities to be copied in all categories. Categories in order: + - log control file + - log files + - Aria tables + - other ("miscellaneous") files + */ + int unsafe_copy_step(const backup_sink &sink) noexcept + { + bool copy_done= false; + + /* If control file is always the first file copied and there is only + one, it is never included in the "steps remaining" calculation. + Should the order be changed, the calculation needs to be updated for + the control file as well. */ + if (have_control_file) + { + bool already_copied= control_file_copied.exchange(true); + if (!already_copied) + { + if (copy_control_file(sink) != 0) + return -1; + copy_done= true; + } + } + + Copy_from_list copy_from_list(copy_done); + if (copy_from_list(log_files, log_files_copied, + [this, sink](const std::string &path) noexcept + { + return copy_file(sink, path); + }) != 0) + return -1; + + return copy_from_list.remaining(); + } + + int end(bool /*abort*/) noexcept { - int ret_val= perform_backup(sink); translog_enable_purge(); - return ret_val; + return 0; } private: const backup_target ⌖ // FIXME: pass to member functions @@ -82,20 +218,45 @@ namespace const int datadir_fd; #endif static constexpr const char zerobuf[511]{}; - static constexpr const char *const data_exts[]= - {"MAD", "MAI", "MYD", "MYI", "frm", "par"}; + /* All file suffixes are 4 characters long (dot and 3 letter extension) */ + static constexpr size_t suffix_len= 4; + static const char* data_ext; + static const char* index_ext; + static const LEX_CSTRING log_file_prefix; + static const LEX_CSTRING tmp_prefix; + static const char* misc_exts[]; + static const char* control_file_name; + using dir_name = std::string; using dir_contents = std::vector; - using database_dir = std::pair; - std::vector database_dirs; + using database_dir = std::pair; + using database_dirs = std::vector; + /* Transactional tables with checksum */ + database_dirs dml_safe_tables; + /* All other Aria tables */ + database_dirs unsafe_tables; + /* Aria log files */ std::vector log_files; + std::vector misc_files; + /* directories in which misc files are */ + std::vector misc_dirs; + bool have_control_file = false; + bool safe_files_copied = false; - int perform_backup(const backup_sink &sink) noexcept - { - return scan_datadir() || copy_databases(sink) || - copy_control_file(sink) || translog_flush(translog_get_horizon()) || - copy_logs(sink); - } + /* Refer to a string stored elsewhere */ + using dir_ref= std::string_view; + using tablename_ref= std::string_view; + using table_ref= std::pair; + using table_list= std::vector; + + /* Flattened versions of dml_safe_tables and unsafe_tables. */ + table_list dml_safe_table_list; + table_list unsafe_tables_list; + std::atomic dml_safe_tables_copied {0}; + std::atomic unsafe_tables_copied {0}; + std::atomic log_files_copied {0}; + std::atomic misc_files_copied {0}; + std::atomic control_file_copied {false}; ATTRIBUTE_COLD ATTRIBUTE_NOINLINE static int dir_error(const char *name) noexcept @@ -104,7 +265,7 @@ namespace return 1; } - int scan_datadir() noexcept + int scan_dbdirs() noexcept { const char *base_dir= maria_data_root; MY_DIR *dir_info= my_dir(base_dir, MYF(MY_WANT_STAT)); @@ -119,10 +280,6 @@ namespace if ((fail= scan_database_dir(fi.name)) != 0) goto func_exit; } - else if (!strncmp(fi.name, C_STRING_WITH_LEN("aria_log."))) - log_files.emplace_back(fi.name); - else if (!strcmp(fi.name, "aria_log_control")) - have_control_file = true; func_exit: my_dirend(dir_info); return fail; @@ -135,34 +292,117 @@ namespace MY_DIR *dir_info= my_dir(dir_path.c_str(), MYF(MY_WANT_STAT)); if (!dir_info) return dir_error(dir_path.c_str()); - std::vector files_to_backup; + int fail= 0; + dir_contents safe; + dir_contents unsafe; for (const fileinfo &fi : st_::span{dir_info->dir_entry, dir_info->number_of_files}) - if (is_db_file(fi.name)) - files_to_backup.emplace_back(fi.name); - if (!files_to_backup.empty()) - database_dirs.emplace_back(dir_name, std::move(files_to_backup)); + { + const char* filename= fi.name; + size_t filename_len = strlen(filename); + if (filename_len >= suffix_len) + { + const char* suffix = filename + filename_len - suffix_len; + if(match_suffix(suffix, index_ext)) + { + if (!is_tmp_table(filename)) + { + auto is_safe = is_safe_table(dir_name, filename); + if (std::holds_alternative(is_safe)) + { + std::string table_name(filename, filename_len - suffix_len); + if (std::get(is_safe)) + safe.push_back(std::move(table_name)); + else + unsafe.push_back(std::move(table_name)); + } + else + { + fail= std::get(is_safe); + goto finish; + } + } + } + else if (match_misc_ext(suffix) || !strcmp(filename, "db.opt")) + { + if(misc_dirs.empty() || misc_dirs.back() != dir_name) + misc_dirs.emplace_back(dir_name); + misc_files.push_back(std::string(dir_name) + "/" + filename); + } + } + } + if(!fail) + { + if (!safe.empty()) + dml_safe_tables.emplace_back(dir_name, std::move(safe)); + if (!unsafe.empty()) + unsafe_tables.emplace_back(dir_name, std::move(unsafe)); + } + finish: my_dirend(dir_info); - return 0; + return fail; + } + + static bool is_tmp_table(const char* filename) noexcept + { + return begins_with(filename, tmp_prefix); + } + + void flatten_table_lists() noexcept + { + flatten_table_list(dml_safe_tables, dml_safe_table_list); + flatten_table_list(unsafe_tables, unsafe_tables_list); } - int copy_databases(const backup_sink &sink) noexcept + static void flatten_table_list(const database_dirs& dirs, table_list& list) noexcept { - for (const database_dir& dir : database_dirs) + for (const database_dir& dir : dirs) { - if (sink.stream == sink.NO_STREAM && - ensure_target_subdir(dir.first.c_str())) - { - my_error(ER_CANT_CREATE_FILE, MYF(0), dir.first.c_str(), errno); - return 1; - } - if (copy_database(sink, dir) != 0) - return 1; + for (const std::string& table : dir.second) + list.emplace_back(dir.first, table); } + } + + int scan_logs() noexcept + { + const char *base_dir= maria_data_root; + MY_DIR *dir_info= my_dir(base_dir, MYF(MY_WANT_STAT)); + if (!dir_info) + return dir_error(base_dir); + for (const fileinfo &fi : + st_::span{dir_info->dir_entry, + dir_info->number_of_files}) + if (begins_with(fi.name, log_file_prefix)) + log_files.emplace_back(fi.name); + else if (strcmp(fi.name, "aria_log_control") == 0) + have_control_file = true; + my_dirend(dir_info); return 0; } + bool ensure_target_dirs() noexcept + { + using string = std::string; + std::vector dirs; + for (const database_dir &dir : dml_safe_tables) + dirs.push_back(&dir.first); + for (const database_dir &dir : unsafe_tables) + dirs.push_back(&dir.first); + for (const string &dir: misc_dirs) + dirs.push_back(&dir); + std::sort(dirs.begin(), dirs.end(), + [](const string *a, const string *b) { return *a < *b; }); + auto dirs_end = std::unique(dirs.begin(), dirs.end(), + [](const string *a, const string *b) { return *a == *b; }); + for (auto it = dirs.begin(); it != dirs_end; ++it) + { + if (ensure_target_subdir((*it)->c_str())) + return true; + } + return false; + } + /* Create directory in the target directory if it does not exist. Return 0 on success, non-0 on failure. Set errno in case of failure @@ -187,33 +427,74 @@ namespace return 0; } - int copy_database(const backup_sink &sink, const database_dir& dir) - noexcept + /* Returns result or error code. */ + std::variant is_safe_table(const char* dir_name, const char* myi_file_name) { - for (const std::string& file : dir.second) + ARIA_TABLE_CAPABILITIES cap; +#ifndef _WIN32 + std::string path= std::string(dir_name) + "/" + myi_file_name; + File fd= openat(datadir_fd, path.c_str(), O_RDONLY); + if (fd < 0) { - std::string file_path= dir.first + "/" + file; - if (copy_file(sink, file_path.c_str()) != 0) - return 1; + my_errno= errno; + my_error(ER_CANT_OPEN_FILE, MYF(0), path.c_str(), errno); } - return 0; +#else + std::string path= std::string(maria_data_root) + "/" + + dir_name + "/" + myi_file_name; + File fd= my_open(path.c_str(), O_RDONLY, MYF(MY_WME)); +#endif + if (fd < 0) + { + return my_errno; + } + std::variant result; + mysql_mutex_lock(&THR_LOCK_maria); + int fail = aria_get_capabilities(fd, myi_file_name, &cap); + if (fail) + { + my_error(ER_FILE_CORRUPT, MYF(0), path.c_str()); + result= fail; + goto end; + } + result = cap.transactional && cap.checksum; + aria_free_capabilities(&cap); +end: + mysql_mutex_unlock(&THR_LOCK_maria); + close(fd); + return result; + } + + int copy_table(const backup_sink &sink, const table_ref& table) noexcept + { + dir_ref dir_name = table.first; + tablename_ref table_name = table.second; + std::string index_path; + index_path.reserve(dir_name.size() + table_name.size() + 5); + index_path= dir_name; + index_path += "/"; + size_t dir_part_length = index_path.size(); + index_path.resize(index_path.size() + table_name.size()); + std::copy(table_name.begin(), table_name.end(), + index_path.begin() + dir_part_length); + std::string data_path; + data_path.reserve(dir_name.size() + table_name.size() + 5); + data_path= index_path; + index_path+= index_ext; + data_path+= data_ext; + return copy_file(sink, index_path) || copy_file(sink, data_path); } int copy_control_file(const backup_sink &sink) noexcept { if (!have_control_file) return 0; - return copy_file(sink, "aria_log_control"); + return copy_file(sink, control_file_name); } - int copy_logs(const backup_sink &sink) noexcept + int copy_file(const backup_sink &sink, const std::string &path) const noexcept { - for (const std::string& file : log_files) - { - if (copy_file(sink, file.c_str()) != 0) - return 1; - } - return 0; + return copy_file(sink, path.c_str()); } int copy_file(const backup_sink &sink, const char *path) const noexcept @@ -323,17 +604,18 @@ namespace #endif } + /* Match if suffix is one of the "other" extensions we need to copy */ + static bool match_misc_ext(const char* suffix) noexcept; + static bool match_suffix(const char* suffix1, const char* suffix2) noexcept + { + return memcmp(suffix1, + suffix2, + suffix_len) == 0; + } - static bool is_db_file(const char* file_name) noexcept + static bool begins_with(const char* str, const LEX_CSTRING &prefix) noexcept { - const char *suffix= strrchr(file_name, '.'); - if (!suffix) - return false; - suffix++; - for (const char *ext : data_exts) - if (!strcmp(suffix, ext)) - return true; - return !strcmp(file_name, "db.opt"); + return strncmp(str, prefix.str, prefix.length) == 0; } #ifdef _WIN32 @@ -344,46 +626,97 @@ namespace } #endif }; + + + const LEX_CSTRING Aria_backup::log_file_prefix {C_STRING_WITH_LEN("aria_log.")}; + const LEX_CSTRING Aria_backup::tmp_prefix {C_STRING_WITH_LEN("#sql")}; + const char* Aria_backup::data_ext (MARIA_NAME_DEXT); + const char* Aria_backup::index_ext (MARIA_NAME_IEXT); + const char* Aria_backup::control_file_name {"aria_log_control"}; + + /* TODO: .frm failes are not Aria-specific, .MYD and .MYI are MyISAM files; + they are copied here as a stop-gap */ + const char* Aria_backup::misc_exts[] {".MYD", ".MYI", ".frm"}; + + bool Aria_backup::match_misc_ext(const char* suffix) noexcept + { + return std::find_if(std::begin(misc_exts), std::end(misc_exts), + [suffix](const char* ext) { + return match_suffix(suffix, ext); + }) != std::end(misc_exts); + } } void *aria_backup_start(THD *thd, const backup_target *target, backup_phase phase, const backup_sink *sink) noexcept { - if (phase != BACKUP_PHASE_NO_COMMIT) - return sink->ha_data; - assert(!sink->ha_data); - Aria_backup *aria_backup= new Aria_backup(*target, *sink); - if (!aria_backup->is_initialized()) + Aria_backup *aria_backup {}; + if (phase == BACKUP_PHASE_START) { - delete aria_backup; - return reinterpret_cast(-1); + assert(!sink->ha_data); + aria_backup= new Aria_backup(*target); + if (!aria_backup->is_initialized()) + { + delete aria_backup; + goto error; + } + return aria_backup; } - return aria_backup; + assert (sink->ha_data != reinterpret_cast(-1)); + aria_backup= static_cast(sink->ha_data); + assert(aria_backup); + switch(phase) + { + case BACKUP_PHASE_NO_DDL: + if (aria_backup->start_copy_dml_safe(*sink)) + goto error; + break; + case BACKUP_PHASE_NO_COMMIT: + if (aria_backup->start_copy_unsafe()) + goto error; + break; + default: + break; + } + return sink->ha_data; +error: + return reinterpret_cast(-1); } -int aria_backup_step(THD*, const backup_target*, backup_phase, - const backup_sink*) noexcept +int aria_backup_step(THD*, const backup_target*, backup_phase phase, + const backup_sink *sink) noexcept { - // FIXME: implement the copying here - return 0; + assert (sink->ha_data != reinterpret_cast(-1)); + Aria_backup *aria_backup= static_cast(sink->ha_data); + assert(aria_backup); + switch (phase) + { + case BACKUP_PHASE_NO_DDL: +#if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others + tc_purge(); + tdc_purge(true); +#endif + return aria_backup->dml_safe_copy_step(*sink); + case BACKUP_PHASE_NO_COMMIT: + return aria_backup->unsafe_copy_step(*sink); + default: + return 0; + } } int aria_backup_end(THD *thd, const backup_target*, backup_phase phase, const backup_sink *sink) noexcept { + assert (sink->ha_data != reinterpret_cast(-1)); Aria_backup *aria_backup= static_cast(sink->ha_data); switch (phase) { case BACKUP_PHASE_NO_COMMIT: assert(aria_backup); -#if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others - tc_purge(); - tdc_purge(true); -#endif - return aria_backup->end(*sink); + return aria_backup->end(thd); case BACKUP_PHASE_FINISH: delete aria_backup; /* fall through */ default: return 0; } -} +} \ No newline at end of file From 863dede0ef2c9ca167e3cfc1b8aa82975880adaa Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Tue, 9 Jun 2026 16:28:07 +0200 Subject: [PATCH 08/14] Fix Windows build issue. --- storage/maria/ma_backup.cc | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/storage/maria/ma_backup.cc b/storage/maria/ma_backup.cc index 6a34bc7250984..c8fdfbb5e625d 100644 --- a/storage/maria/ma_backup.cc +++ b/storage/maria/ma_backup.cc @@ -75,18 +75,18 @@ namespace if (idx < list.size()) { if (copy_action(list[idx]) != 0) - return 1; + return true; m_copy_done= true; - m_remaining+= list.size() - idx - 1U; + m_remaining+= static_cast(list.size() - idx - 1U); } } else { size_t current_copied= copied.load(std::memory_order_relaxed); if (current_copied < list.size()) - m_remaining+= list.size() - current_copied; + m_remaining+= static_cast(list.size() - current_copied); } - return 0; + return false; } }; @@ -629,7 +629,7 @@ namespace const LEX_CSTRING Aria_backup::log_file_prefix {C_STRING_WITH_LEN("aria_log.")}; - const LEX_CSTRING Aria_backup::tmp_prefix {C_STRING_WITH_LEN("#sql")}; + const LEX_CSTRING Aria_backup::tmp_prefix {C_STRING_WITH_LEN(tmp_file_prefix)}; const char* Aria_backup::data_ext (MARIA_NAME_DEXT); const char* Aria_backup::index_ext (MARIA_NAME_IEXT); const char* Aria_backup::control_file_name {"aria_log_control"}; @@ -667,6 +667,12 @@ void *aria_backup_start(THD *thd, const backup_target *target, assert(aria_backup); switch(phase) { +#if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others + case BACKUP_PHASE_NO_DML_NON_TRANS: + tc_purge(); + tdc_purge(true); + break; +#endif case BACKUP_PHASE_NO_DDL: if (aria_backup->start_copy_dml_safe(*sink)) goto error; @@ -692,10 +698,6 @@ int aria_backup_step(THD*, const backup_target*, backup_phase phase, switch (phase) { case BACKUP_PHASE_NO_DDL: -#if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others - tc_purge(); - tdc_purge(true); -#endif return aria_backup->dml_safe_copy_step(*sink); case BACKUP_PHASE_NO_COMMIT: return aria_backup->unsafe_copy_step(*sink); From e8b376c44a69210575c11b5280bb8ecbadd05095 Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Tue, 9 Jun 2026 20:34:50 +0200 Subject: [PATCH 09/14] Incorporate changes from code review. --- storage/maria/ma_backup.cc | 76 +++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/storage/maria/ma_backup.cc b/storage/maria/ma_backup.cc index c8fdfbb5e625d..020722a9b4b0e 100644 --- a/storage/maria/ma_backup.cc +++ b/storage/maria/ma_backup.cc @@ -220,12 +220,14 @@ namespace static constexpr const char zerobuf[511]{}; /* All file suffixes are 4 characters long (dot and 3 letter extension) */ static constexpr size_t suffix_len= 4; - static const char* data_ext; - static const char* index_ext; - static const LEX_CSTRING log_file_prefix; - static const LEX_CSTRING tmp_prefix; - static const char* misc_exts[]; - static const char* control_file_name; + static constexpr const char* data_ext {MARIA_NAME_DEXT}; + static constexpr const char* index_ext {MARIA_NAME_IEXT}; + static constexpr LEX_CSTRING log_file_prefix {C_STRING_WITH_LEN("aria_log.")}; + static constexpr LEX_CSTRING tmp_prefix {C_STRING_WITH_LEN(tmp_file_prefix)}; + /* TODO: .frm failes are not Aria-specific, .MYD and .MYI are MyISAM files; + they are copied here as a stop-gap */ + static constexpr const char* misc_exts[] {".MYD", ".MYI", ".frm"}; + static constexpr const char* control_file_name {"aria_log_control"}; using dir_name = std::string; using dir_contents = std::vector; using database_dir = std::pair; @@ -288,7 +290,7 @@ namespace int scan_database_dir(const char* dir_name) noexcept { const char* base_dir = maria_data_root; - std::string dir_path = std::string(base_dir) + "/" + dir_name; + const std::string dir_path= build_path(base_dir, dir_name); MY_DIR *dir_info= my_dir(dir_path.c_str(), MYF(MY_WANT_STAT)); if (!dir_info) return dir_error(dir_path.c_str()); @@ -328,7 +330,7 @@ namespace { if(misc_dirs.empty() || misc_dirs.back() != dir_name) misc_dirs.emplace_back(dir_name); - misc_files.push_back(std::string(dir_name) + "/" + filename); + misc_files.push_back(build_path(dir_name, filename)); } } } @@ -410,7 +412,7 @@ namespace int ensure_target_subdir(const char* name) noexcept { #ifdef _WIN32 - std::string dir_path= targetPath() + "/" + name; + const std::string dir_path= build_path(targetPath(), name); if (!CreateDirectory(dir_path.c_str(), nullptr)) { DWORD err = GetLastError(); @@ -472,11 +474,8 @@ namespace std::string index_path; index_path.reserve(dir_name.size() + table_name.size() + 5); index_path= dir_name; - index_path += "/"; - size_t dir_part_length = index_path.size(); - index_path.resize(index_path.size() + table_name.size()); - std::copy(table_name.begin(), table_name.end(), - index_path.begin() + dir_part_length); + index_path += '/'; + index_path.append(table_name.begin(), table_name.end()); std::string data_path; data_path.reserve(dir_name.size() + table_name.size() + 5); data_path= index_path; @@ -540,11 +539,11 @@ namespace close(src_fd); return ret_val; #else - std::string src_path= std::string(maria_data_root) + "/" + path; + const std::string src_path= build_path(maria_data_root, path); if (sink.stream == sink.NO_STREAM) { - std::string dest_path= targetPath() + "/" + path; + const std::string dest_path= build_path(targetPath(), path); if (!CopyFileEx(src_path.c_str(), dest_path.c_str(), nullptr, nullptr, nullptr, COPY_FILE_NO_BUFFERING)) { @@ -605,7 +604,14 @@ namespace } /* Match if suffix is one of the "other" extensions we need to copy */ - static bool match_misc_ext(const char* suffix) noexcept; + static bool match_misc_ext(const char* suffix) noexcept + { + return std::find_if(std::begin(misc_exts), std::end(misc_exts), + [suffix](const char* ext) { + return match_suffix(suffix, ext); + }) != std::end(misc_exts); + } + static bool match_suffix(const char* suffix1, const char* suffix2) noexcept { return memcmp(suffix1, @@ -618,33 +624,26 @@ namespace return strncmp(str, prefix.str, prefix.length) == 0; } + static std::string build_path(const char *base_path, const char *filename) noexcept + { + std::string path; + const size_t base_len= strlen(base_path); + const size_t filename_len= strlen(filename); + path.reserve(base_len + filename_len + 1); + path.append(base_path, base_len); + path+= '/'; + path.append(filename, filename_len); + return path; + } + #ifdef _WIN32 /** @return the target directory path */ - std::string targetPath() const + const char* targetPath() const { - return std::string(target.path); + return target.path; } #endif }; - - - const LEX_CSTRING Aria_backup::log_file_prefix {C_STRING_WITH_LEN("aria_log.")}; - const LEX_CSTRING Aria_backup::tmp_prefix {C_STRING_WITH_LEN(tmp_file_prefix)}; - const char* Aria_backup::data_ext (MARIA_NAME_DEXT); - const char* Aria_backup::index_ext (MARIA_NAME_IEXT); - const char* Aria_backup::control_file_name {"aria_log_control"}; - - /* TODO: .frm failes are not Aria-specific, .MYD and .MYI are MyISAM files; - they are copied here as a stop-gap */ - const char* Aria_backup::misc_exts[] {".MYD", ".MYI", ".frm"}; - - bool Aria_backup::match_misc_ext(const char* suffix) noexcept - { - return std::find_if(std::begin(misc_exts), std::end(misc_exts), - [suffix](const char* ext) { - return match_suffix(suffix, ext); - }) != std::end(misc_exts); - } } void *aria_backup_start(THD *thd, const backup_target *target, @@ -669,6 +668,7 @@ void *aria_backup_start(THD *thd, const backup_target *target, { #if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others case BACKUP_PHASE_NO_DML_NON_TRANS: + /* FIXME: Would be better to selectively purge only the tables we need. */ tc_purge(); tdc_purge(true); break; From 0e18555ee4e47adddf63f0a08ca751538db3c97f Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Wed, 10 Jun 2026 09:58:27 +0200 Subject: [PATCH 10/14] Fix Windows crash. Enable backup_server_restore for Windows. --- storage/maria/ma_backup.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/storage/maria/ma_backup.cc b/storage/maria/ma_backup.cc index 020722a9b4b0e..ae97ceae5f3c2 100644 --- a/storage/maria/ma_backup.cc +++ b/storage/maria/ma_backup.cc @@ -463,7 +463,11 @@ namespace aria_free_capabilities(&cap); end: mysql_mutex_unlock(&THR_LOCK_maria); +#ifndef _WIN32 close(fd); +#else + my_close(fd, MYF(0)); +#endif return result; } From 2c50745aeb95b6fb1f497b2862a272810a5377d9 Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Wed, 10 Jun 2026 12:02:32 +0200 Subject: [PATCH 11/14] Fix concurrent BACKUP_PHASE_FINISH issue. --- storage/maria/ma_backup.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/maria/ma_backup.cc b/storage/maria/ma_backup.cc index ae97ceae5f3c2..e789e80ffd39a 100644 --- a/storage/maria/ma_backup.cc +++ b/storage/maria/ma_backup.cc @@ -30,6 +30,7 @@ #include "span.h" #include #include +#include #include /* @@ -725,4 +726,4 @@ int aria_backup_end(THD *thd, const backup_target*, backup_phase phase, default: return 0; } -} \ No newline at end of file +} From b38451a022948dfbd42f0b48d29e9f38d45a41a6 Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Fri, 12 Jun 2026 08:09:48 +0200 Subject: [PATCH 12/14] Add a test for Aria concurrent backup with many tables --- .../backup/backup_aria_concurrent.result | 20 +++ .../suite/backup/backup_aria_concurrent.test | 145 ++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 mysql-test/suite/backup/backup_aria_concurrent.result create mode 100644 mysql-test/suite/backup/backup_aria_concurrent.test diff --git a/mysql-test/suite/backup/backup_aria_concurrent.result b/mysql-test/suite/backup/backup_aria_concurrent.result new file mode 100644 index 0000000000000..07b2341253c2c --- /dev/null +++ b/mysql-test/suite/backup/backup_aria_concurrent.result @@ -0,0 +1,20 @@ +15 30 7500 +Back up the database +BACKUP SERVER TO '$target_directory' 4 CONCURRENT; +Restore the database +# restart: --datadir=MYSQLTEST_VARDIR/some_directory +Check contents after restore +SELECT COUNT(*) FROM table_checks; +COUNT(*) +400 +SELECT * FROM table_checks WHERE sum_id <> 15; +tbl_name sum_id str_len blob_len num_rows +SELECT * FROM table_checks WHERE str_len <> 30; +tbl_name sum_id str_len blob_len num_rows +SELECT * FROM table_checks WHERE blob_len <> 7500; +tbl_name sum_id str_len blob_len num_rows +SELECT * FROM table_checks WHERE num_rows <> 5; +tbl_name sum_id str_len blob_len num_rows +Restart database in original data directory +# restart +Clean up diff --git a/mysql-test/suite/backup/backup_aria_concurrent.test b/mysql-test/suite/backup/backup_aria_concurrent.test new file mode 100644 index 0000000000000..fd77d05a31a80 --- /dev/null +++ b/mysql-test/suite/backup/backup_aria_concurrent.test @@ -0,0 +1,145 @@ + +--source include/have_aria.inc + +--disable_query_log + + +DELIMITER //; +CREATE PROCEDURE populate_data(IN t_name VARCHAR(64), IN num_rows INT) +BEGIN + DECLARE i INT DEFAULT 1; + SET @query = CONCAT('INSERT INTO ', t_name, ' (id, str_val, blob_val) VALUES (?, ?, ?)'); + PREPARE stmt FROM @query; + + WHILE i <= num_rows DO + SET @str = CONCAT('_row_', i); + # Generate a predictable but repeating blob based on the row index + SET @blb = REPEAT(CHAR(97 + (i % 26)), 1500); + EXECUTE stmt USING i, @str, @blb; + SET i = i + 1; + END WHILE; + + DEALLOCATE PREPARE stmt; +END// +DELIMITER ;// + +# Create this many tables transactional and non-transactional each +let $tab_num= 200; +let $num_rows= 5; + +let $i = 1; +while ($i <= $tab_num) { + + # Create Transactional Table + + let $tr=0; + while ($tr <= 1) { + + let $suff= _$i; + let $table_name= ta_tr$tr$suff; + + eval CREATE TABLE $table_name ( + id INT PRIMARY KEY, + str_val VARCHAR(255), + blob_val BLOB, + INDEX idx_str (str_val) + ) ENGINE=Aria TRANSACTIONAL=$tr; + + eval CALL populate_data('$table_name', $num_rows); + + inc $tr; + } + + inc $i; +} + +--enable_query_log + +# All tables have the same data, so we query only one for reference + +let $sum_id= `SELECT SUM(id) FROM ta_tr0_1`; +let $str_len= `SELECT SUM(LENGTH(str_val)) FROM ta_tr0_1`; +let $blob_len= `SELECT SUM(LENGTH(blob_val)) FROM ta_tr0_1`; + +echo $sum_id $str_len $blob_len; + +--let $target_directory=$MYSQLTEST_VARDIR/some_directory + +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory + +--echo Back up the database +evalp BACKUP SERVER TO '$target_directory' 4 CONCURRENT; + +--echo Restore the database +--let $restart_parameters=--datadir=$target_directory +--source include/restart_mysqld.inc + +--echo Check contents after restore + +--disable_query_log +CREATE TEMPORARY TABLE table_checks ( + tbl_name VARCHAR(64), + sum_id INT, + str_len INT, + blob_len INT, + num_rows INT +) ENGINE=MEMORY; + +let $i = 1; +while ($i <= $tab_num) { + let $tr=0; + while ($tr <= 1) { + + let $suff= _$i; + let $table_name= ta_tr$tr$suff; + + let $r_sum_id= `SELECT SUM(id) FROM $table_name`; + let $r_str_len= `SELECT SUM(LENGTH(str_val)) FROM $table_name`; + let $r_blob_len= `SELECT SUM(LENGTH(blob_val)) FROM $table_name`; + let $r_num_rows= `SELECT COUNT(*) FROM $table_name`; + + eval INSERT INTO table_checks VALUES ('$table_name', $r_sum_id, $r_str_len, $r_blob_len, $r_num_rows); + + inc $tr; + } + inc $i; +} + +--enable_query_log + +SELECT COUNT(*) FROM table_checks; + +# We expect results in the table to always match the results captured before the BACKUP +# Returned rowsets should be empty +eval SELECT * FROM table_checks WHERE sum_id <> $sum_id; +eval SELECT * FROM table_checks WHERE str_len <> $str_len; +eval SELECT * FROM table_checks WHERE blob_len <> $blob_len; +eval SELECT * FROM table_checks WHERE num_rows <> $num_rows; + +--echo Restart database in original data directory +--let $restart_parameters= +--source include/restart_mysqld.inc + +--echo Clean up + +--disable_query_log + +let $i = 1; +while ($i <= $tab_num) { + let $tr=0; + while ($tr <= 1) { + let $suff= _$i; + let $table_name= ta_tr$tr$suff; + eval DROP TABLE $table_name; + inc $tr; + } + inc $i; +} + +DROP PROCEDURE populate_data; + +--enable_query_log + +--rmdir $MYSQLTEST_VARDIR/some_directory From 6a7286ebc375980ce0c1e03b8ea4a68aa9bfc7bc Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Tue, 23 Jun 2026 11:09:39 +0200 Subject: [PATCH 13/14] MDEV-39988 Copy non-ACID engine files in BACKUP SERVER Move copying of common files and MyISAM files out of Aria plugim Fix incorrect handling of non-default aria_log_dir_path. --- .../suite/backup/backup_aria_concurrent.test | 4 +- .../suite/backup/backup_aria_log_dir.result | 53 ++++ .../suite/backup/backup_aria_log_dir.test | 74 +++++ mysql-test/suite/backup/backup_nonacid.result | 56 ++++ mysql-test/suite/backup/backup_nonacid.test | 45 +++ sql/sql_backup.cc | 283 ++++++++++++++++- sql/sql_backup_interface.h | 146 +++++++-- storage/maria/ma_backup.cc | 287 +++++------------- 8 files changed, 700 insertions(+), 248 deletions(-) create mode 100644 mysql-test/suite/backup/backup_aria_log_dir.result create mode 100644 mysql-test/suite/backup/backup_aria_log_dir.test create mode 100644 mysql-test/suite/backup/backup_nonacid.result create mode 100644 mysql-test/suite/backup/backup_nonacid.test diff --git a/mysql-test/suite/backup/backup_aria_concurrent.test b/mysql-test/suite/backup/backup_aria_concurrent.test index fd77d05a31a80..868644fa2daec 100644 --- a/mysql-test/suite/backup/backup_aria_concurrent.test +++ b/mysql-test/suite/backup/backup_aria_concurrent.test @@ -30,8 +30,6 @@ let $num_rows= 5; let $i = 1; while ($i <= $tab_num) { - # Create Transactional Table - let $tr=0; while ($tr <= 1) { @@ -142,4 +140,4 @@ DROP PROCEDURE populate_data; --enable_query_log ---rmdir $MYSQLTEST_VARDIR/some_directory +--rmdir $target_directory diff --git a/mysql-test/suite/backup/backup_aria_log_dir.result b/mysql-test/suite/backup/backup_aria_log_dir.result new file mode 100644 index 0000000000000..beeb6db9b8684 --- /dev/null +++ b/mysql-test/suite/backup/backup_aria_log_dir.result @@ -0,0 +1,53 @@ +# restart: --aria-log-dir-path=MYSQLTEST_VARDIR/log_directory +CREATE TABLE t ( +id INT PRIMARY KEY, +str_val VARCHAR(255), +blob_val BLOB, +INDEX idx_str (str_val) +) ENGINE=Aria TRANSACTIONAL=1; +CREATE PROCEDURE populate_data(IN num_rows INT) +BEGIN +DECLARE i INT DEFAULT 0; +WHILE i < num_rows DO +SET @str = CONCAT('_row_', i); +SET @blb = REPEAT(CHAR(97 + (i % 26)), 1500); +INSERT INTO t (id, str_val, blob_val) VALUES (i, @str, @blb); +SET i = i + 1; +END WHILE; +END// +CALL populate_data(10000); +SELECT COUNT(*) from t; +COUNT(*) +10000 +SELECT SUM(id) FROM t; +SUM(id) +49995000 +SELECT SUM(LENGTH(str_val)) FROM t; +SUM(LENGTH(str_val)) +88890 +SELECT SUM(LENGTH(blob_val)) FROM t; +SUM(LENGTH(blob_val)) +15000000 +Back up the database +BACKUP SERVER TO '$target_directory'; +Restore the database +# restart: --datadir=MYSQLTEST_VARDIR/some_directory +Check contents after restore +SELECT COUNT(*) from t; +COUNT(*) +10000 +SELECT SUM(id) FROM t; +SUM(id) +49995000 +SELECT SUM(LENGTH(str_val)) FROM t; +SUM(LENGTH(str_val)) +88890 +SELECT SUM(LENGTH(blob_val)) FROM t; +SUM(LENGTH(blob_val)) +15000000 +Restart database in original log and data directories +# restart: --aria-log-dir-path=MYSQLTEST_VARDIR/log_directory +Clean up +DROP PROCEDURE populate_data; +DROP TABLE t; +# restart diff --git a/mysql-test/suite/backup/backup_aria_log_dir.test b/mysql-test/suite/backup/backup_aria_log_dir.test new file mode 100644 index 0000000000000..4c12212062b92 --- /dev/null +++ b/mysql-test/suite/backup/backup_aria_log_dir.test @@ -0,0 +1,74 @@ +--source include/have_aria.inc + +--let $log_directory=$MYSQLTEST_VARDIR/log_directory +--let $target_directory=$MYSQLTEST_VARDIR/some_directory + +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $log_directory +--error 0,1 +--rmdir $target_directory + +--mkdir $log_directory + +--let $orig_restart_parameters=--aria-log-dir-path=$log_directory +--let $restart_parameters=$orig_restart_parameters +--source include/restart_mysqld.inc + +CREATE TABLE t ( + id INT PRIMARY KEY, + str_val VARCHAR(255), + blob_val BLOB, + INDEX idx_str (str_val) +) ENGINE=Aria TRANSACTIONAL=1; + +--disable_warnings +DELIMITER //; +CREATE PROCEDURE populate_data(IN num_rows INT) +BEGIN + DECLARE i INT DEFAULT 0; + WHILE i < num_rows DO + SET @str = CONCAT('_row_', i); + SET @blb = REPEAT(CHAR(97 + (i % 26)), 1500); + INSERT INTO t (id, str_val, blob_val) VALUES (i, @str, @blb); + SET i = i + 1; + END WHILE; +END// +DELIMITER ;// +--enable_warnings + +CALL populate_data(10000); + +SELECT COUNT(*) from t; +SELECT SUM(id) FROM t; +SELECT SUM(LENGTH(str_val)) FROM t; +SELECT SUM(LENGTH(blob_val)) FROM t; + +--echo Back up the database +evalp BACKUP SERVER TO '$target_directory'; + +--echo Restore the database +--let $restart_parameters=--datadir=$target_directory +--source include/restart_mysqld.inc + +--echo Check contents after restore + +SELECT COUNT(*) from t; +SELECT SUM(id) FROM t; +SELECT SUM(LENGTH(str_val)) FROM t; +SELECT SUM(LENGTH(blob_val)) FROM t; + +--echo Restart database in original log and data directories +--let $restart_parameters=$orig_restart_parameters +--source include/restart_mysqld.inc + +--echo Clean up + +DROP PROCEDURE populate_data; +DROP TABLE t; + +--let $restart_parameters= +--source include/restart_mysqld.inc + +--rmdir $target_directory +--rmdir $log_directory diff --git a/mysql-test/suite/backup/backup_nonacid.result b/mysql-test/suite/backup/backup_nonacid.result new file mode 100644 index 0000000000000..9027dd735bedf --- /dev/null +++ b/mysql-test/suite/backup/backup_nonacid.result @@ -0,0 +1,56 @@ +CREATE TABLE t_archive (id int unsigned) ENGINE=ARCHIVE; +INSERT INTO t_archive VALUES (2), (3), (5), (7), (11); +CREATE DATABASE d; +CREATE TABLE d.t_csv (id int unsigned NOT NULL) ENGINE=CSV; +INSERT INTO d.t_csv VALUES (4), (26), (41), (60), (83), (109); +CREATE TABLE t_myisam1 (id int unsigned) ENGINE=MyISAM; +INSERT INTO t_myisam1 VALUES (1), (1), (2), (3), (5), (8); +CREATE TABLE t_myisam2 (id int unsigned) ENGINE=MyISAM; +INSERT INTO t_myisam2 VALUES (13), (21), (34), (55), (89), (144); +CREATE TABLE t_mrg (id int unsigned) ENGINE=MRG_MyISAM UNION=(t_myisam1, t_myisam2); +BACKUP SERVER TO '$target_directory'; +# restart: --datadir=MYSQLTEST_VARDIR/some_directory +SELECT * FROM t_archive ORDER BY id; +id +2 +3 +5 +7 +11 +SELECT * FROM d.t_csv ORDER BY id; +id +4 +26 +41 +60 +83 +109 +SELECT * FROM t_myisam1 ORDER BY id; +id +1 +1 +2 +3 +5 +8 +SELECT * FROM t_mrg ORDER BY id; +id +1 +1 +2 +3 +5 +8 +13 +21 +34 +55 +89 +144 +# restart +DROP TABLE t_archive; +DROP TABLE d.t_csv; +DROP TABLE t_myisam1; +DROP TABLE t_myisam2; +DROP TABLE t_mrg; +DROP DATABASE d; diff --git a/mysql-test/suite/backup/backup_nonacid.test b/mysql-test/suite/backup/backup_nonacid.test new file mode 100644 index 0000000000000..eac863a191685 --- /dev/null +++ b/mysql-test/suite/backup/backup_nonacid.test @@ -0,0 +1,45 @@ +--source include/have_csv.inc +--source include/have_archive.inc + +CREATE TABLE t_archive (id int unsigned) ENGINE=ARCHIVE; +INSERT INTO t_archive VALUES (2), (3), (5), (7), (11); + +CREATE DATABASE d; +CREATE TABLE d.t_csv (id int unsigned NOT NULL) ENGINE=CSV; +INSERT INTO d.t_csv VALUES (4), (26), (41), (60), (83), (109); + +CREATE TABLE t_myisam1 (id int unsigned) ENGINE=MyISAM; +INSERT INTO t_myisam1 VALUES (1), (1), (2), (3), (5), (8); + +CREATE TABLE t_myisam2 (id int unsigned) ENGINE=MyISAM; +INSERT INTO t_myisam2 VALUES (13), (21), (34), (55), (89), (144); + +CREATE TABLE t_mrg (id int unsigned) ENGINE=MRG_MyISAM UNION=(t_myisam1, t_myisam2); + +--let $target_directory=$MYSQLTEST_VARDIR/some_directory + +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory + +evalp BACKUP SERVER TO '$target_directory'; + +--let $restart_parameters=--datadir=$target_directory +--source include/restart_mysqld.inc + +SELECT * FROM t_archive ORDER BY id; +SELECT * FROM d.t_csv ORDER BY id; +SELECT * FROM t_myisam1 ORDER BY id; +SELECT * FROM t_mrg ORDER BY id; + +--let $restart_parameters= +--source include/restart_mysqld.inc + +DROP TABLE t_archive; +DROP TABLE d.t_csv; +DROP TABLE t_myisam1; +DROP TABLE t_myisam2; +DROP TABLE t_mrg; +DROP DATABASE d; + +--rmdir $target_directory diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc index fddb9f2f175da..eb5fd1e5e75af 100644 --- a/sql/sql_backup.cc +++ b/sql/sql_backup.cc @@ -24,6 +24,12 @@ #include "tpool.h" #include "aligned.h" +#include +#include +#include + +static constexpr const char zerobuf[511]{}; + #if defined __linux__ || defined __FreeBSD__ using copying_step= ssize_t(int,int,size_t,off_t*); template @@ -168,7 +174,79 @@ static ssize_t pread_write(IF_WIN(const native_file_handle&,int) in_fd, #ifdef __APPLE__ /* The inline copy_entire_file() invokes fcopyfile() */ #elif defined _WIN32 -/* CopyFileEx() should be used */ +/** Copy entire file. + @param src_path path file file to copy + @param dst_path path of file to copy to + @param target backup target + @param sink worker context + @return error code (non-positive) + @retval 0 on success + @note Wrapper for CopyFileExA, will report error using my_error */ +extern "C" +int copy_entire_file(const char *src_path, const char *dst_path, + const struct backup_target *target, + const struct backup_sink *sink) +{ + if (sink->stream == sink->NO_STREAM) + { + std::string full_dst_path= build_path(target->path, dst_path); + if (!CopyFileEx(src_path, full_dst_path.c_str(), nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_CREATE_FILE, MYF(0), full_dst_path.c_str(), errno); + return 1; + } + } + else + { + HANDLE src, dst{sink->stream}; + for (;;) + { + src= CreateFile(src_path, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (src != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + + my_osmaperr(GetLastError()); + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), src_path, errno); + return -1; + } + + LARGE_INTEGER li; + if (!GetFileSizeEx(src, &li)) + { + write_error: + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_WRITE, MYF(0), dst_path, errno); + if (src != INVALID_HANDLE_VALUE) + CloseHandle(src); + return -1; + } + + if (backup_stream_start(dst, dst_path, 0644, li.QuadPart, nullptr, 0) || + backup_stream_append_plain(src, dst, 0, li.QuadPart)) + goto write_error; + + if (size_t pad= size_t(li.LowPart) & 511) + if (backup_stream_write(dst, zerobuf, 512 - pad)) + goto write_error; + if (!CloseHandle(src)) + { + src= INVALID_HANDLE_VALUE; + goto write_error; + } + } + return 0; +} #else /** Copy a file (whole content). @param src source file descriptor @@ -181,6 +259,56 @@ extern "C" int copy_entire_file(int src, int dst) } #endif +#ifndef _WIN32 +/** Copy an entire file to target. +@param src_fd source file descriptor +@param target backup target +@return error code (non-positive) +@retval 0 on success +@note Any intermediate directories must already exist in the target. */ +# ifdef __cplusplus +extern "C" +# endif +int copy_fd_to_target(int src_fd, + const struct backup_target *target, + const char *path, + const struct backup_sink *sink) +{ + int ret_val= 0; + int tgt_fd{sink->stream}; + if (tgt_fd == sink->NO_STREAM) + { + tgt_fd= openat(target->fd, path, + O_CREAT | O_EXCL | O_WRONLY, 0666); + if (tgt_fd < 0) + { + my_error(ER_CANT_CREATE_FILE, MYF(0), path, errno); + ret_val = 1; + goto finish; + } + ret_val= copy_entire_file(src_fd, tgt_fd); + if (ret_val | close(tgt_fd)) + { + write_error: + my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); + ret_val = 1; + } + } + else + { + uint64_t end= uint64_t(lseek(src_fd, 0, SEEK_END)); + if (backup_stream_start(tgt_fd, path, 0644, end, nullptr, 0) || + backup_stream_append(src_fd, tgt_fd, 0, end)) + goto write_error; + if (size_t pad= size_t(end) & 511) + if (backup_stream_write(tgt_fd, zerobuf, 512 - pad)) + goto write_error; + } +finish: + return ret_val; +} +#endif + /** Copy a portion of a file. @param src source file descriptor @param dst target to append src to @@ -215,6 +343,153 @@ extern "C" int copy_file(IF_WIN(const native_file_handle&,int) src, return int(ret); } +/** Ensure a file can be copied to a subdirectory in target. +May create the subdirectory. +@param target backup target +@param name subdirectory name +@return error code (non-positive) +@retval 0 on success +@note If the directory is created, the directory containing it must + already exist: nested directory creation is not supported. */ +extern "C" int ensure_target_subdir(const struct backup_target *target, + const char* name) +{ +#ifdef _WIN32 + const std::string dir_path= build_path(target->path, name); + if (!CreateDirectory(dir_path.c_str(), nullptr)) + { + DWORD err = GetLastError(); + if (err != ERROR_ALREADY_EXISTS) + { + my_osmaperr(err); + return 1; + } + } +#else + if (mkdirat(target->fd, name, 0777) != 0) + return (errno != EEXIST); +#endif + return 0; +} + +/** Copy entire file from data directory target, preserving path. +@param path relative path of file +@param target backup target +@param sink worker context +@return error code (non-positive) +@retval 0 on success +@note The file will be copied to the same path relative to + target directory. Any intermediate directories must + already exist in the target. */ +extern "C" int copy_datafile_to_target(const char *path, + const struct backup_target *target, + const backup_sink *sink) +{ +#ifndef _WIN32 + int src_fd = open(path, O_RDONLY); + if (src_fd < 0) + { + my_error(ER_CANT_OPEN_FILE, MYF(0), path, errno); + return 1; + } + int ret_val= copy_fd_to_target(src_fd, target, path, sink); + close(src_fd); + return ret_val; +#else + return copy_entire_file(path, path, target, sink); +#endif +} + +/* all extensions have the same length, adjust if that changes */ +static constexpr size_t ext_len= 4; + +/* Files not copied by plugin backup implementations: files managed by +SQL layer and miscellaneous engine files to be copied bunde DDL lock */ +static constexpr const char* misc_exts[] {".frm", ".MYD", ".MYI", ".MRG", + ".ARM", ".ARZ", ".CSM", ".CSV"}; +static constexpr const char db_opt_name[] {"db.opt"}; +static constexpr size_t db_opt_len= sizeof(db_opt_name) - 1; + +static bool match_ext(const char* ext1, const char* ext2) noexcept +{ + return memcmp(ext1, + ext2, + ext_len) == 0; +} + +static bool match_misc_ext(const char* file_ext) noexcept +{ + return std::find_if(std::begin(misc_exts), std::end(misc_exts), + [file_ext](const char* misc_ext) { + return match_ext(file_ext, misc_ext); + }) != std::end(misc_exts); +} + +static bool is_db_opt(const char* filename, size_t filename_len) +{ + return filename_len == db_opt_len && + memcmp(filename, db_opt_name, db_opt_len) == 0; +} + +static bool is_misc_file(const char* filename) +{ + size_t filename_len= strlen(filename); + if (filename_len < ext_len) + return false; + const char *file_ext = filename + filename_len - ext_len; + return match_misc_ext(file_ext) || is_db_opt(filename, filename_len); +} + +std::string build_path(const char *base_path, const char *filename) noexcept +{ + std::string path; + const size_t base_len= strlen(base_path); + const size_t filename_len= strlen(filename); + path.reserve(base_len + filename_len + 1); + path.append(base_path, base_len); + path+= '/'; + path.append(filename, filename_len); + return path; +} + +static bool copy_misc_files(const backup_target *target, + const backup_sink *sink) +{ + Dir_scan datadir(".", MYF(MY_WANT_STAT)); + if (datadir.is_error()) + return true; + std::unordered_set ensured_dirs; + int error= datadir.for_each([target, sink, &ensured_dirs](const fileinfo &fi) + { + if ((fi.mystat->st_mode & S_IFMT) == S_IFDIR) + { + const char* dir_name= fi.name; + if(sink->stream == sink->NO_STREAM && + ensured_dirs.insert(dir_name).second) + { + int fail= ensure_target_subdir(target, dir_name); + if (fail) + return fail; + } + Dir_scan dbdir(dir_name, MYF(0)); + if (dbdir.is_error()) + return 1; + return dbdir.for_each([target, sink, dir_name](const fileinfo &fi) + { + if (is_misc_file(fi.name)) + { + const std::string path= build_path(dir_name, fi.name); + return copy_datafile_to_target(path.c_str(), target, sink); + } + return 0; + }); + } + return 0; + }); + return error != 0; +} + + /** Append to the configuration file. @param target backup target directory @param config the configuration file snippet to append @@ -510,6 +785,12 @@ bool Sql_cmd_backup::execute(THD *thd) } backup_phase_start: target_phase->phase= backup_phase(phase); + + if (phase == BACKUP_PHASE_NO_DDL) + fail= copy_misc_files(&target_phase->target, &target_phase->sink); + if (fail) + break; + fail= plugin_foreach_with_mask(thd, backup_start, MYSQL_STORAGE_ENGINE_PLUGIN, PLUGIN_IS_DELETED|PLUGIN_IS_READY, diff --git a/sql/sql_backup_interface.h b/sql/sql_backup_interface.h index 94892ba7591ab..07209b8c7af20 100644 --- a/sql/sql_backup_interface.h +++ b/sql/sql_backup_interface.h @@ -13,7 +13,16 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ +#include +#include +#include + +#ifdef __cplusplus +# include +#endif + struct backup_target; +struct backup_sink; /** A payload chunk in a sparse file that is being streamed */ struct backup_chunk @@ -25,14 +34,30 @@ struct backup_chunk }; #ifdef _WIN32 -/* Use CopyFileEx() to copy entire files */ +/** Copy entire file. + @param src_path path file file to copy + @param dst_path path of file to copy to + @param target backup target + @param sink worker context + @return error code (non-positive) + @retval 0 on success + @note Wrapper for CopyFileExA, will report error using my_error */ +# ifdef __cplusplus +extern "C" +# endif +int copy_entire_file(const char *src_path, + const char *dst_path, + const struct backup_target *target, + const struct backup_sink *sink); + struct native_file_handle; -#elif defined __APPLE__ +#else +# if defined __APPLE__ /* You should invoke fclonefileat(2) manually before attempting copy_entire_file() or copy_file() */ -# include -# include -# include +# include +# include +# include /** Copy an entire file. @param src source file descriptor @param dst target to append src to @@ -42,20 +67,39 @@ inline int copy_entire_file(int src, int dst) { return fcopyfile(src, dst, NULL, COPYFILE_ALL | COPYFILE_CLONE); } -#else -# ifdef __cplusplus +# else +# ifdef __cplusplus extern "C" -# endif +# endif /** Copy an entire file. @param src source file descriptor @param dst target to append src to @return error code (non-positive) @retval 0 on success */ int copy_entire_file(int src, int dst); +# endif + +/** Copy an entire file to target. +@param src_fd source file descriptor +@param target backup target +@param path target file path +@param sink worker context +@return error code (non-positive) +@retval 0 on success +@note Any intermediate directories must already exist in the target. */ +# ifdef __cplusplus +extern "C" +# endif +int copy_fd_to_target(int src_fd, + const struct backup_target *target, + const char *path, + const struct backup_sink *sink); + #endif #ifdef __cplusplus extern "C" +{ #endif /** Copy a portion of a file. @param src source file descriptor @@ -68,9 +112,29 @@ int copy_file(IF_WIN(const native_file_handle&,int) src, IF_WIN(const native_file_handle&,int) dst, uint64_t start, uint64_t end); -#ifdef __cplusplus -extern "C" -#endif +/** Ensure a file can be copied to a subdirectory in target. +May create the subdirectory. +@param target backup target +@param name subdirectory name +@return error code (non-positive) +@retval 0 on success +@note If the directory is created, the directory containing it must + already exist: nested directory creation is not supported. */ +int ensure_target_subdir(const struct backup_target *target, const char* name); + +/** Copy entire file from data directory target, preserving path. +@param path relative path of file +@param target backup target +@param sink worker context +@return error code (non-positive) +@retval 0 on success +@note The file will be copied to the same path relative to + target directory. Any intermediate directories must + already exist in the target. */ +int copy_datafile_to_target(const char *path, + const struct backup_target *target, + const struct backup_sink *sink); + /** Append to the configuration file. @param target backup target directory @param config the configuration file snippet to append @@ -80,9 +144,7 @@ extern "C" int backup_config_append(IF_WIN(const char*, int) target, const char *config, size_t size); -#ifdef __cplusplus -extern "C" -#endif + /** Append to the configuration file. @param target backup stream @param config the configuration file snippet to append @@ -92,9 +154,6 @@ extern "C" int backup_stream_config(IF_WIN(HANDLE, int) stream, const char *config, size_t size); -#ifdef __cplusplus -extern "C" -#endif /** Start streaming a file. @param target backup target @param name file name @@ -108,9 +167,6 @@ int backup_stream_start(IF_WIN(HANDLE, int) stream, const char *name, mode_t mode, uint64_t size, const struct backup_chunk *chunks, size_t n_chunks); -#ifdef __cplusplus -extern "C" -#endif /** Write data to a stream. @param stream backup stream @@ -122,9 +178,6 @@ extern "C" int backup_stream_write(IF_WIN(HANDLE, int) stream, const void *buf, size_t size); -#ifdef __cplusplus -extern "C" -#endif /** Append a file snippet to the stream, after a corresponding call to backup_stream_start(). @@ -143,9 +196,6 @@ int backup_stream_append(IF_WIN(const native_file_handle&,int) src, uint64_t start, uint64_t end); #ifdef __linux__ -# ifdef __cplusplus -extern "C" -# endif /** Append an immutable snippet of a file to the stream, allowing Linux sendfile(2) to be invoked. @@ -168,11 +218,49 @@ int backup_stream_append_async(int src, int stream, #endif #ifdef _WIN32 -# ifdef __cplusplus -extern "C" -# endif int backup_stream_append_plain(HANDLE src, HANDLE stream, uint64_t start, uint64_t end); #else # define backup_stream_append_plain backup_stream_append #endif + +#ifdef __cplusplus +} // extern "C" + + /* RAII wrapper for my_dir() */ +class Dir_scan +{ +public: + explicit Dir_scan(const char* path, myf flags) noexcept + { + dir_info= my_dir(path, flags); + if (!dir_info) + { + my_error(ER_CANT_READ_DIR, MYF(0), path, my_errno); + } + } + ~Dir_scan() noexcept + { + my_dirend(dir_info); + } + bool is_error() const noexcept + { + return !dir_info; + } + template + int for_each(Fn fn) const noexcept + { + for (size_t i= 0; i < dir_info->number_of_files; i++) + { + if (fn(dir_info->dir_entry[i]) != 0) + return 1; + } + return 0; + } +private: + MY_DIR *dir_info {nullptr}; +}; + +std::string build_path(const char *base_path, const char *filename) noexcept; + +#endif diff --git a/storage/maria/ma_backup.cc b/storage/maria/ma_backup.cc index e789e80ffd39a..4b58dacba8843 100644 --- a/storage/maria/ma_backup.cc +++ b/storage/maria/ma_backup.cc @@ -98,9 +98,9 @@ namespace Aria_backup(const backup_target &target) noexcept : target(target) #ifndef _WIN32 - , datadir_fd(open(maria_data_root, O_DIRECTORY)) + , logdir_fd(open(maria_data_root, O_DIRECTORY)) { - if (datadir_fd < 0) + if (logdir_fd < 0) { my_error(ER_CANT_READ_DIR, MYF(0), maria_data_root, errno); return; @@ -114,7 +114,7 @@ namespace bool is_initialized() const noexcept { #ifndef _WIN32 - return datadir_fd >= 0; + return logdir_fd >= 0; #else return true; #endif // _WIN32 @@ -123,17 +123,17 @@ namespace ~Aria_backup() noexcept { #ifndef _WIN32 - if (datadir_fd >= 0) - close(datadir_fd); + if (logdir_fd >= 0) + close(logdir_fd); #endif // _WIN32 } - bool start_copy_dml_safe(const backup_sink &sink) noexcept + bool start_copy_dml_safe(const backup_sink *sink) noexcept { if (scan_dbdirs()) return true; flatten_table_lists(); - if (sink.stream == sink.NO_STREAM) + if (sink->stream == sink->NO_STREAM) return ensure_target_dirs(); return false; } @@ -147,7 +147,7 @@ namespace /* Copy an Aria table that is safe to be copied while concurrent DML is in progress. */ - int dml_safe_copy_step(const backup_sink &sink) noexcept + int dml_safe_copy_step(const backup_sink *sink) noexcept { Copy_from_list copy_from_list; auto copy_table_action= [this, sink](const table_ref &table) noexcept @@ -160,12 +160,6 @@ namespace if (copy_from_list(unsafe_tables_list, unsafe_tables_copied, copy_table_action) != 0) return -1; - if (copy_from_list(misc_files, misc_files_copied, - [this, sink](const std::string &path) noexcept - { - return copy_file(sink, path); - }) != 0) - return -1; return copy_from_list.remaining(); } @@ -178,7 +172,7 @@ namespace - Aria tables - other ("miscellaneous") files */ - int unsafe_copy_step(const backup_sink &sink) noexcept + int unsafe_copy_step(const backup_sink *sink) noexcept { bool copy_done= false; @@ -201,7 +195,7 @@ namespace if (copy_from_list(log_files, log_files_copied, [this, sink](const std::string &path) noexcept { - return copy_file(sink, path); + return copy_log_file(sink, path.c_str()); }) != 0) return -1; @@ -216,19 +210,15 @@ namespace private: const backup_target ⌖ // FIXME: pass to member functions #ifndef _WIN32 - const int datadir_fd; + const int logdir_fd; #endif - static constexpr const char zerobuf[511]{}; - /* All file suffixes are 4 characters long (dot and 3 letter extension) */ - static constexpr size_t suffix_len= 4; + /* File extensions are 4 characters long (dot and 3 letter extension) */ + static constexpr size_t ext_len= 4; static constexpr const char* data_ext {MARIA_NAME_DEXT}; static constexpr const char* index_ext {MARIA_NAME_IEXT}; static constexpr LEX_CSTRING log_file_prefix {C_STRING_WITH_LEN("aria_log.")}; static constexpr LEX_CSTRING tmp_prefix {C_STRING_WITH_LEN(tmp_file_prefix)}; - /* TODO: .frm failes are not Aria-specific, .MYD and .MYI are MyISAM files; - they are copied here as a stop-gap */ - static constexpr const char* misc_exts[] {".MYD", ".MYI", ".frm"}; - static constexpr const char* control_file_name {"aria_log_control"}; + static constexpr LEX_CSTRING control_file_name {C_STRING_WITH_LEN("aria_log_control")}; using dir_name = std::string; using dir_contents = std::vector; using database_dir = std::pair; @@ -239,9 +229,6 @@ namespace database_dirs unsafe_tables; /* Aria log files */ std::vector log_files; - std::vector misc_files; - /* directories in which misc files are */ - std::vector misc_dirs; bool have_control_file = false; bool safe_files_copied = false; @@ -258,7 +245,6 @@ namespace std::atomic dml_safe_tables_copied {0}; std::atomic unsafe_tables_copied {0}; std::atomic log_files_copied {0}; - std::atomic misc_files_copied {0}; std::atomic control_file_copied {false}; ATTRIBUTE_COLD ATTRIBUTE_NOINLINE @@ -270,10 +256,9 @@ namespace int scan_dbdirs() noexcept { - const char *base_dir= maria_data_root; - MY_DIR *dir_info= my_dir(base_dir, MYF(MY_WANT_STAT)); + MY_DIR *dir_info= my_dir(".", MYF(MY_WANT_STAT)); if (!dir_info) - return dir_error(base_dir); + return dir_error("."); int fail= 0; for (const fileinfo &fi : st_::span{dir_info->dir_entry, @@ -290,11 +275,9 @@ namespace int scan_database_dir(const char* dir_name) noexcept { - const char* base_dir = maria_data_root; - const std::string dir_path= build_path(base_dir, dir_name); - MY_DIR *dir_info= my_dir(dir_path.c_str(), MYF(MY_WANT_STAT)); + MY_DIR *dir_info= my_dir(dir_name, MYF(MY_WANT_STAT)); if (!dir_info) - return dir_error(dir_path.c_str()); + return dir_error(dir_name); int fail= 0; dir_contents safe; dir_contents unsafe; @@ -302,19 +285,20 @@ namespace st_::span{dir_info->dir_entry, dir_info->number_of_files}) { - const char* filename= fi.name; - size_t filename_len = strlen(filename); - if (filename_len >= suffix_len) + const LEX_CSTRING filename {fi.name, strlen(fi.name)}; + if (filename.length >= ext_len) { - const char* suffix = filename + filename_len - suffix_len; - if(match_suffix(suffix, index_ext)) + /* Length of filename without extension. */ + size_t base_filename_len= filename.length - ext_len; + const char* suffix = filename.str + base_filename_len; + if(match_ext(suffix, index_ext)) { if (!is_tmp_table(filename)) { - auto is_safe = is_safe_table(dir_name, filename); + auto is_safe = is_safe_table(dir_name, filename.str); if (std::holds_alternative(is_safe)) { - std::string table_name(filename, filename_len - suffix_len); + std::string table_name(filename.str, base_filename_len); if (std::get(is_safe)) safe.push_back(std::move(table_name)); else @@ -327,12 +311,6 @@ namespace } } } - else if (match_misc_ext(suffix) || !strcmp(filename, "db.opt")) - { - if(misc_dirs.empty() || misc_dirs.back() != dir_name) - misc_dirs.emplace_back(dir_name); - misc_files.push_back(build_path(dir_name, filename)); - } } } if(!fail) @@ -347,7 +325,7 @@ namespace return fail; } - static bool is_tmp_table(const char* filename) noexcept + static bool is_tmp_table(const LEX_CSTRING &filename) noexcept { return begins_with(filename, tmp_prefix); } @@ -376,10 +354,13 @@ namespace for (const fileinfo &fi : st_::span{dir_info->dir_entry, dir_info->number_of_files}) - if (begins_with(fi.name, log_file_prefix)) - log_files.emplace_back(fi.name); - else if (strcmp(fi.name, "aria_log_control") == 0) + { + const LEX_CSTRING filename {fi.name, strlen(fi.name)}; + if (begins_with(filename, log_file_prefix)) + log_files.emplace_back(LEX_STRING_WITH_LEN(filename)); + else if (is_control_file_name(filename)) have_control_file = true; + } my_dirend(dir_info); return 0; } @@ -392,8 +373,6 @@ namespace dirs.push_back(&dir.first); for (const database_dir &dir : unsafe_tables) dirs.push_back(&dir.first); - for (const string &dir: misc_dirs) - dirs.push_back(&dir); std::sort(dirs.begin(), dirs.end(), [](const string *a, const string *b) { return *a < *b; }); auto dirs_end = std::unique(dirs.begin(), dirs.end(), @@ -412,43 +391,18 @@ namespace */ int ensure_target_subdir(const char* name) noexcept { -#ifdef _WIN32 - const std::string dir_path= build_path(targetPath(), name); - if (!CreateDirectory(dir_path.c_str(), nullptr)) - { - DWORD err = GetLastError(); - if (err != ERROR_ALREADY_EXISTS) - { - my_osmaperr(err); - return 1; - } - } -#else - if (mkdirat(target.fd, name, 0777) != 0) - return (errno != EEXIST); -#endif - return 0; + return ::ensure_target_subdir(&target, name); } /* Returns result or error code. */ std::variant is_safe_table(const char* dir_name, const char* myi_file_name) { ARIA_TABLE_CAPABILITIES cap; -#ifndef _WIN32 - std::string path= std::string(dir_name) + "/" + myi_file_name; - File fd= openat(datadir_fd, path.c_str(), O_RDONLY); - if (fd < 0) - { - my_errno= errno; - my_error(ER_CANT_OPEN_FILE, MYF(0), path.c_str(), errno); - } -#else - std::string path= std::string(maria_data_root) + "/" + - dir_name + "/" + myi_file_name; + std::string path= build_path(dir_name, myi_file_name); File fd= my_open(path.c_str(), O_RDONLY, MYF(MY_WME)); -#endif if (fd < 0) { + my_error(ER_CANT_OPEN_FILE, MYF(0), path.c_str(), my_errno); return my_errno; } std::variant result; @@ -464,15 +418,11 @@ namespace aria_free_capabilities(&cap); end: mysql_mutex_unlock(&THR_LOCK_maria); -#ifndef _WIN32 - close(fd); -#else my_close(fd, MYF(0)); -#endif return result; } - int copy_table(const backup_sink &sink, const table_ref& table) noexcept + int copy_table(const backup_sink *sink, const table_ref& table) noexcept { dir_ref dir_name = table.first; tablename_ref table_name = table.second; @@ -486,159 +436,65 @@ namespace data_path= index_path; index_path+= index_ext; data_path+= data_ext; - return copy_file(sink, index_path) || copy_file(sink, data_path); + return copy_table_file(sink, index_path) || copy_table_file(sink, data_path); } - int copy_control_file(const backup_sink &sink) noexcept + int copy_control_file(const backup_sink *sink) noexcept { if (!have_control_file) return 0; - return copy_file(sink, control_file_name); + return copy_log_file(sink, control_file_name.str); } - int copy_file(const backup_sink &sink, const std::string &path) const noexcept + int copy_table_file(const backup_sink *sink, + const std::string &path) const noexcept { - return copy_file(sink, path.c_str()); + return copy_table_file(sink, path.c_str()); } - int copy_file(const backup_sink &sink, const char *path) const noexcept + int copy_table_file(const backup_sink *sink, const char *path) const noexcept + { + return ::copy_datafile_to_target(path, &target, sink); + } + + int copy_log_file(const backup_sink *sink, const char *filename) { #ifndef _WIN32 - int ret_val = 0; - int src_fd = openat(datadir_fd, path, O_RDONLY); + int src_fd = openat(logdir_fd, filename, O_RDONLY); if (src_fd < 0) { - my_error(ER_CANT_OPEN_FILE, MYF(0), path, errno); + my_error(ER_CANT_OPEN_FILE, MYF(0), + build_path(maria_data_root, filename).c_str(), + errno); return 1; } - int tgt_fd{sink.stream}; - if (tgt_fd == sink.NO_STREAM) - { - tgt_fd= openat(target.fd, path, - O_CREAT | O_EXCL | O_WRONLY, 0666); - if (tgt_fd < 0) - { - my_error(ER_CANT_CREATE_FILE, MYF(0), path, errno); - ret_val = 1; - goto finish; - } - ret_val= copy_entire_file(src_fd, tgt_fd); - if (ret_val | close(tgt_fd)) - { - write_error: - my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); - ret_val = 1; - } - } - else - { - uint64_t end= uint64_t(lseek(src_fd, 0, SEEK_END)); - if (backup_stream_start(tgt_fd, path, 0644, end, nullptr, 0) || - backup_stream_append(src_fd, tgt_fd, 0, end)) - goto write_error; - if (size_t pad= size_t(end) & 511) - if (backup_stream_write(tgt_fd, zerobuf, 512 - pad)) - goto write_error; - } - finish: + int ret_val= copy_fd_to_target(src_fd, &target, filename, sink); close(src_fd); return ret_val; #else - const std::string src_path= build_path(maria_data_root, path); - - if (sink.stream == sink.NO_STREAM) - { - const std::string dest_path= build_path(targetPath(), path); - if (!CopyFileEx(src_path.c_str(), dest_path.c_str(), nullptr, nullptr, nullptr, - COPY_FILE_NO_BUFFERING)) - { - my_osmaperr(GetLastError()); - my_error(ER_CANT_CREATE_FILE, MYF(0), dest_path.c_str(), errno); - return 1; - } - } - else - { - HANDLE src, dst{sink.stream}; - for (;;) - { - src= CreateFile(src_path.c_str(), GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - my_win_file_secattr(), OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, nullptr); - if (src != INVALID_HANDLE_VALUE) - break; - switch (GetLastError()) { - case ERROR_SHARING_VIOLATION: - case ERROR_LOCK_VIOLATION: - std::this_thread::sleep_for(std::chrono::seconds(1)); - continue; - } - - my_osmaperr(GetLastError()); - my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), src_path.c_str(), errno); - return -1; - } - - LARGE_INTEGER li; - if (!GetFileSizeEx(src, &li)) - { - write_error: - my_osmaperr(GetLastError()); - my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); - if (src != INVALID_HANDLE_VALUE) - CloseHandle(src); - return -1; - } - - if (backup_stream_start(dst, path, 0644, li.QuadPart, nullptr, 0) || - backup_stream_append_plain(src, dst, 0, li.QuadPart)) - goto write_error; - - if (size_t pad= size_t(li.LowPart) & 511) - if (backup_stream_write(dst, zerobuf, 512 - pad)) - goto write_error; - if (!CloseHandle(src)) - { - src= INVALID_HANDLE_VALUE; - goto write_error; - } - } - return 0; + return copy_entire_file(build_path(maria_data_root, filename).c_str(), + filename, &target, sink); #endif } - /* Match if suffix is one of the "other" extensions we need to copy */ - static bool match_misc_ext(const char* suffix) noexcept - { - return std::find_if(std::begin(misc_exts), std::end(misc_exts), - [suffix](const char* ext) { - return match_suffix(suffix, ext); - }) != std::end(misc_exts); - } - - static bool match_suffix(const char* suffix1, const char* suffix2) noexcept + static bool match_ext(const char* ext1, const char* ext2) noexcept { - return memcmp(suffix1, - suffix2, - suffix_len) == 0; + return memcmp(ext1, + ext2, + ext_len) == 0; } - static bool begins_with(const char* str, const LEX_CSTRING &prefix) noexcept + static bool begins_with(const LEX_CSTRING &str, const LEX_CSTRING &prefix) noexcept { - return strncmp(str, prefix.str, prefix.length) == 0; + if (str.length < prefix.length) + return false; + return memcmp(str.str, prefix.str, prefix.length) == 0; } - static std::string build_path(const char *base_path, const char *filename) noexcept + static bool is_control_file_name(const LEX_CSTRING &str) { - std::string path; - const size_t base_len= strlen(base_path); - const size_t filename_len= strlen(filename); - path.reserve(base_len + filename_len + 1); - path.append(base_path, base_len); - path+= '/'; - path.append(filename, filename_len); - return path; + return str.length == control_file_name.length && + memcmp(str.str, control_file_name.str, control_file_name.length) == 0; } #ifdef _WIN32 @@ -673,13 +529,14 @@ void *aria_backup_start(THD *thd, const backup_target *target, { #if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others case BACKUP_PHASE_NO_DML_NON_TRANS: - /* FIXME: Would be better to selectively purge only the tables we need. */ + /* FIXME: Would be better to selectively purge only the tables we need. + To be fixed in MDEV-39987. */ tc_purge(); tdc_purge(true); break; #endif case BACKUP_PHASE_NO_DDL: - if (aria_backup->start_copy_dml_safe(*sink)) + if (aria_backup->start_copy_dml_safe(sink)) goto error; break; case BACKUP_PHASE_NO_COMMIT: @@ -703,9 +560,9 @@ int aria_backup_step(THD*, const backup_target*, backup_phase phase, switch (phase) { case BACKUP_PHASE_NO_DDL: - return aria_backup->dml_safe_copy_step(*sink); + return aria_backup->dml_safe_copy_step(sink); case BACKUP_PHASE_NO_COMMIT: - return aria_backup->unsafe_copy_step(*sink); + return aria_backup->unsafe_copy_step(sink); default: return 0; } From d4b2048efe6863be2b80108674416f36b935d530 Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Wed, 24 Jun 2026 11:49:26 +0200 Subject: [PATCH 14/14] Refactor and simplify code given all Aria tables files are now copied in the same phase. --- storage/maria/ma_backup.cc | 197 +++++++------------------------------ 1 file changed, 37 insertions(+), 160 deletions(-) diff --git a/storage/maria/ma_backup.cc b/storage/maria/ma_backup.cc index 4b58dacba8843..cce4fe60dc644 100644 --- a/storage/maria/ma_backup.cc +++ b/storage/maria/ma_backup.cc @@ -31,7 +31,6 @@ #include #include #include -#include /* Implementation of functions declatred in ma_backup.h: @@ -40,58 +39,6 @@ namespace { - /* Utility class to implement the "backup step" interface when - processing several lists. It implements the logic where an item - is processed (copied) from the first list which has available - items, and a "remaining" counter accumulates the number of - items remaining to be processed on all lists, regardless of - whether an item from that list was processed or not. */ - class Copy_from_list - { - int m_remaining {0}; - bool m_copy_done; - public: - Copy_from_list(bool copy_done= false) noexcept - : m_copy_done(copy_done) - { - } - - bool copy_done() const noexcept - { - return m_copy_done; - } - - int remaining() const noexcept - { - return m_remaining; - } - - template - bool operator()(const T &list, std::atomic &copied, - Fn copy_action) noexcept - { - if(!m_copy_done) - { - size_t idx= copied.fetch_add(1, std::memory_order_relaxed); - if (idx < list.size()) - { - if (copy_action(list[idx]) != 0) - return true; - m_copy_done= true; - m_remaining+= static_cast(list.size() - idx - 1U); - } - } - else - { - size_t current_copied= copied.load(std::memory_order_relaxed); - if (current_copied < list.size()) - m_remaining+= static_cast(list.size() - current_copied); - } - return false; - } - }; - - class Aria_backup { public: @@ -149,18 +96,11 @@ namespace is in progress. */ int dml_safe_copy_step(const backup_sink *sink) noexcept { - Copy_from_list copy_from_list; - auto copy_table_action= [this, sink](const table_ref &table) noexcept - { - return copy_table(sink, table); - }; - if (copy_from_list(dml_safe_table_list, dml_safe_tables_copied, - copy_table_action) != 0) - return -1; - if (copy_from_list(unsafe_tables_list, unsafe_tables_copied, - copy_table_action) != 0) - return -1; - return copy_from_list.remaining(); + return copy_from_list_step(flat_table_list, tables_copied, + [this, sink](const table_ref &table) noexcept + { + return copy_table(sink, table); + }); } /* Copy an entity that is not safe to copy if there are concurrent @@ -174,8 +114,6 @@ namespace */ int unsafe_copy_step(const backup_sink *sink) noexcept { - bool copy_done= false; - /* If control file is always the first file copied and there is only one, it is never included in the "steps remaining" calculation. Should the order be changed, the calculation needs to be updated for @@ -187,19 +125,18 @@ namespace { if (copy_control_file(sink) != 0) return -1; - copy_done= true; + size_t current_copied= log_files_copied.load(std::memory_order_relaxed); + return (current_copied < log_files.size()) ? + static_cast(log_files.size() - current_copied) : + 0; } } - Copy_from_list copy_from_list(copy_done); - if (copy_from_list(log_files, log_files_copied, - [this, sink](const std::string &path) noexcept - { - return copy_log_file(sink, path.c_str()); - }) != 0) - return -1; - - return copy_from_list.remaining(); + return copy_from_list_step(log_files, log_files_copied, + [this, sink](const std::string &path) noexcept + { + return copy_log_file(sink, path.c_str()); + }); } int end(bool /*abort*/) noexcept @@ -223,10 +160,8 @@ namespace using dir_contents = std::vector; using database_dir = std::pair; using database_dirs = std::vector; - /* Transactional tables with checksum */ - database_dirs dml_safe_tables; - /* All other Aria tables */ - database_dirs unsafe_tables; + /* Collection of tables to be backed up. */ + database_dirs tables; /* Aria log files */ std::vector log_files; @@ -239,11 +174,8 @@ namespace using table_ref= std::pair; using table_list= std::vector; - /* Flattened versions of dml_safe_tables and unsafe_tables. */ - table_list dml_safe_table_list; - table_list unsafe_tables_list; - std::atomic dml_safe_tables_copied {0}; - std::atomic unsafe_tables_copied {0}; + table_list flat_table_list; + std::atomic tables_copied {0}; std::atomic log_files_copied {0}; std::atomic control_file_copied {false}; @@ -278,9 +210,7 @@ namespace MY_DIR *dir_info= my_dir(dir_name, MYF(MY_WANT_STAT)); if (!dir_info) return dir_error(dir_name); - int fail= 0; - dir_contents safe; - dir_contents unsafe; + dir_contents dir_tables; for (const fileinfo &fi : st_::span{dir_info->dir_entry, dir_info->number_of_files}) @@ -295,34 +225,15 @@ namespace { if (!is_tmp_table(filename)) { - auto is_safe = is_safe_table(dir_name, filename.str); - if (std::holds_alternative(is_safe)) - { - std::string table_name(filename.str, base_filename_len); - if (std::get(is_safe)) - safe.push_back(std::move(table_name)); - else - unsafe.push_back(std::move(table_name)); - } - else - { - fail= std::get(is_safe); - goto finish; - } + dir_tables.emplace_back(filename.str, base_filename_len); } } } } - if(!fail) - { - if (!safe.empty()) - dml_safe_tables.emplace_back(dir_name, std::move(safe)); - if (!unsafe.empty()) - unsafe_tables.emplace_back(dir_name, std::move(unsafe)); - } - finish: + if (!dir_tables.empty()) + tables.emplace_back(dir_name, std::move(dir_tables)); my_dirend(dir_info); - return fail; + return 0; } static bool is_tmp_table(const LEX_CSTRING &filename) noexcept @@ -332,8 +243,7 @@ namespace void flatten_table_lists() noexcept { - flatten_table_list(dml_safe_tables, dml_safe_table_list); - flatten_table_list(unsafe_tables, unsafe_tables_list); + flatten_table_list(tables, flat_table_list); } static void flatten_table_list(const database_dirs& dirs, table_list& list) noexcept @@ -367,59 +277,26 @@ namespace bool ensure_target_dirs() noexcept { - using string = std::string; - std::vector dirs; - for (const database_dir &dir : dml_safe_tables) - dirs.push_back(&dir.first); - for (const database_dir &dir : unsafe_tables) - dirs.push_back(&dir.first); - std::sort(dirs.begin(), dirs.end(), - [](const string *a, const string *b) { return *a < *b; }); - auto dirs_end = std::unique(dirs.begin(), dirs.end(), - [](const string *a, const string *b) { return *a == *b; }); - for (auto it = dirs.begin(); it != dirs_end; ++it) - { - if (ensure_target_subdir((*it)->c_str())) + for (const database_dir &dir : tables) + if(::ensure_target_subdir(&target, dir.first.c_str()) != 0) return true; - } return false; } - /* - Create directory in the target directory if it does not exist. - Return 0 on success, non-0 on failure. Set errno in case of failure - */ - int ensure_target_subdir(const char* name) noexcept - { - return ::ensure_target_subdir(&target, name); - } - - /* Returns result or error code. */ - std::variant is_safe_table(const char* dir_name, const char* myi_file_name) + template + static int copy_from_list_step(const std::vector &list, + std::atomic &copied, + Fn copy_action) { - ARIA_TABLE_CAPABILITIES cap; - std::string path= build_path(dir_name, myi_file_name); - File fd= my_open(path.c_str(), O_RDONLY, MYF(MY_WME)); - if (fd < 0) + size_t idx= copied.fetch_add(1, std::memory_order_relaxed); + if (idx < list.size()) { - my_error(ER_CANT_OPEN_FILE, MYF(0), path.c_str(), my_errno); - return my_errno; - } - std::variant result; - mysql_mutex_lock(&THR_LOCK_maria); - int fail = aria_get_capabilities(fd, myi_file_name, &cap); - if (fail) - { - my_error(ER_FILE_CORRUPT, MYF(0), path.c_str()); - result= fail; - goto end; + if (copy_action(list[idx]) != 0) + return -1; + return static_cast(list.size() - idx - 1U); } - result = cap.transactional && cap.checksum; - aria_free_capabilities(&cap); -end: - mysql_mutex_unlock(&THR_LOCK_maria); - my_close(fd, MYF(0)); - return result; + + return 0; } int copy_table(const backup_sink *sink, const table_ref& table) noexcept