From 81b3ae71537ca4c67ea4d0f740778f1596fd29a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 29 Jun 2026 11:23:30 +0300 Subject: [PATCH 1/3] MDEV-14992 BACKUP SERVER The following SQL statements will be introduced: BACKUP SERVER TO '/path/to/directory' [ 1 CONCURRENT ]; BACKUP SERVER WITH [ 1 CONCURRENT ] 'command'; In place of the 1, any positive number of threads may be specified. For the first variant, '/path/to' must exist and '/path/to/directory' must not exist; that is where the backup will be written to. For the second variant, 'command' must be the name of a script or command that will be executed in a child process. The standard input of that command will be in a format that is compatible with GNU tar --format=oldgnu (and also BSD tar variants that are also part of Microsoft Windows and Apple macOS). The command is expected to optionally compress and encrypt the stream and redirect it to a file on a local or a remote server. The BACKUP SERVER WITH will append an additional argument, a positive base-ten number in ASCII, starting with 1, to identify the current thread. In this way, each concurrent stream can write a separate file. The backup or the first stream will contain a file backup.cnf, which includes parameters needed for restoring the backup. Currently, these are innodb_log_recovery_start and innodb_log_recovery_target. If innodb_log_recovery_target>0, InnoDB will be in read-only mode, not allowing any writes to persistent files other than via the log application. To restore a streaming backup made with BACKUP SERVER WITH, an empty directory needs to be created and all streams be extracted there using the standard tar utility of the operating system, optionally after undoing any encryption or compression that had been added by the backup command. Then, the backup is prepared or MariaDB server started up on the extracted directory, similar to as if the BACKUP SERVER TO statement had been used. Note: The parameter innodb_log_recovery_start in backup.cnf is STRICTLY NECESSARY TO AVOID CORRUPTION! By default, InnoDB crash recovery starts from the latest available log checkpoint. However, for restoring a backup, recovery must start from the checkpoint that was the latest when the backup was started. Starting recovery from a possible later checkpoint will result in a corrupted database! The following will be implemented separately: MDEV-39061 mariadb-backup compatible wrapper script for BACKUP SERVER MDEV-40163 Partial backup and restore MDEV-39091 Back up ENGINE=RocksDB MDEV-39092 Less blocking backup of ENGINE=Aria The implementation introduces a basic driver Sql_cmd_backup, storage engine interfaces, and basic copying of the storage engines InnoDB, Aria, MyISAM, MERGE (MyISAM), Archive, CSV. backup_target: A structured data type to represent a target directory. On Microsoft Windows, we must use directory paths because there is no variant of CopyFileEx() that would work on file handles. backup_sink: Wraps a per-thread output stream as well as storage engine specific context. handlerton::backup_start(), handlerton::backup_end(): Invoked at the start or end of a backup phase, in the thread that executes a BACKUP SERVER statement. handlerton::backup_step(): A backup step that can be invoked from multiple threads concurrently, between the execution of the corresponding handlerton::backup_start() and handlerton::backup_end() of the same phase. copy_entire_file(): A file copying service for POSIX systems. copy_file(): A partial or sparse file-copying service for all systems. backup_stream_append(): Equivalent to copy_file(), but appending to a stream. On Linux, this uses sendfile(2), which assumes that the source data will not be changed before the data has been consumed from the pipe. backup_stream_append_async(): A variant of backup_stream_append() where the source file region is guaranteed to be immutable after the call returns. We must not use Linux sendfile(2) for copying data files that may be modified in place, because it could introduce a race condition between a page write that runs concurrently with a child process that is reading the data from the pipe. InnoDB_backup::context: Backup context, attached to backup_sink so that context can continue to exist between the time a BACKUP SERVER releases all locks and another BACKUP SERVER starts executing, with innodb_backup pointing to the new backup, while the old backup is still being finished. fil_space_t::write_or_backup: Keep track of in-flight page writes and pending backup operation. We must not allow them concurrently, because that could lead into torn pages in the backup. fil_space_t::backup_end: The first page number that is not being backed up (by default 0, to indicate that no backup is in progress). fil_space_t::BACKUP_BATCH_SIZE: The number of preceding pages that will be covered by fil_space_t::backup_end. This is the unit of "page range locking" during InnoDB backup. log_sys.backup: Whether BACKUP SERVER is in progress. The purpose of this is to make BACKUP SERVER prevent the concurrent execution of SET GLOBAL innodb_log_archive=OFF or SET GLOBAL innodb_log_file_size when innodb_log_archive=OFF. log_sys.archived_checkpoint: Keep track of the earliest available checkpoint, corresponding to log_sys.archived_lsn. This reflects SET GLOBAL innodb_log_recovery_start (which is settable now), for incremental backup. buf_flush_list_space(): Check for concurrent backup before writing each page. This is inefficient, but this function may be invoked from multiple threads concurrently, and it cannot be changed easily, especially for fil_crypt_thread(). fil_system.have_all_spaces: Whether all tablespace metadata is guaranteed to be known. To speed up startup, InnoDB does not normally open all tablespace files. --- libmysqld/CMakeLists.txt | 1 + mysql-test/collections/buildbot_suites.bat | 1 + mysql-test/main/backup_server.result | 6 + mysql-test/main/backup_server.test | 10 + mysql-test/main/backup_server_locking.result | 17 + mysql-test/main/backup_server_locking.test | 31 + mysql-test/main/grant_backup_server.result | 27 + mysql-test/main/grant_backup_server.test | 29 + mysql-test/main/mysqld--help.result | 2 +- mysql-test/mariadb-test-run.pl | 1 + .../suite/backup/backup_innodb,debug.rdiff | 16 + .../suite/backup/backup_innodb.combinations | 4 + mysql-test/suite/backup/backup_innodb.result | 60 + mysql-test/suite/backup/backup_innodb.test | 104 ++ .../suite/backup/backup_stream,debug.rdiff | 16 + .../suite/backup/backup_stream.combinations | 4 + mysql-test/suite/backup/backup_stream.result | 35 + mysql-test/suite/backup/backup_stream.test | 85 ++ mysql-test/suite/backup/suite.pm | 19 + .../perfschema/r/max_program_zero.result | 2 +- .../suite/perfschema/r/ortho_iter.result | 2 +- .../perfschema/r/privilege_table_io.result | 2 +- .../r/start_server_disable_idle.result | 2 +- .../r/start_server_disable_stages.result | 2 +- .../r/start_server_disable_statements.result | 2 +- .../start_server_disable_transactions.result | 2 +- .../r/start_server_disable_waits.result | 2 +- .../perfschema/r/start_server_innodb.result | 2 +- .../r/start_server_low_index.result | 2 +- .../r/start_server_low_table_lock.result | 2 +- .../r/start_server_no_account.result | 2 +- .../r/start_server_no_cond_class.result | 2 +- .../r/start_server_no_cond_inst.result | 2 +- .../r/start_server_no_file_class.result | 2 +- .../r/start_server_no_file_inst.result | 2 +- .../perfschema/r/start_server_no_host.result | 2 +- .../perfschema/r/start_server_no_index.result | 2 +- .../perfschema/r/start_server_no_mdl.result | 2 +- .../r/start_server_no_memory_class.result | 2 +- .../r/start_server_no_mutex_class.result | 2 +- .../r/start_server_no_mutex_inst.result | 2 +- ..._server_no_prepared_stmts_instances.result | 2 +- .../r/start_server_no_rwlock_class.result | 2 +- .../r/start_server_no_rwlock_inst.result | 2 +- .../r/start_server_no_setup_actors.result | 2 +- .../r/start_server_no_setup_objects.result | 2 +- .../r/start_server_no_socket_class.result | 2 +- .../r/start_server_no_socket_inst.result | 2 +- .../r/start_server_no_stage_class.result | 2 +- .../r/start_server_no_stages_history.result | 2 +- ...start_server_no_stages_history_long.result | 2 +- .../start_server_no_statements_history.result | 2 +- ...t_server_no_statements_history_long.result | 2 +- .../r/start_server_no_table_hdl.result | 2 +- .../r/start_server_no_table_inst.result | 2 +- .../r/start_server_no_table_lock.result | 2 +- .../r/start_server_no_thread_class.result | 2 +- .../r/start_server_no_thread_inst.result | 2 +- ...tart_server_no_transactions_history.result | 2 +- ...server_no_transactions_history_long.result | 2 +- .../perfschema/r/start_server_no_user.result | 2 +- .../r/start_server_no_waits_history.result | 2 +- .../start_server_no_waits_history_long.result | 2 +- .../perfschema/r/start_server_off.result | 2 +- .../suite/perfschema/r/start_server_on.result | 2 +- .../r/start_server_variables.result | 2 +- .../r/statement_program_lost_inst.result | 2 +- .../suite/sys_vars/r/sysvars_innodb.result | 2 +- sql/CMakeLists.txt | 1 + sql/handler.h | 94 ++ sql/mysqld.cc | 1 + sql/sql_backup.cc | 886 +++++++++++++ sql/sql_backup.h | 53 + sql/sql_backup_interface.h | 178 +++ sql/sql_command.h | 1 + sql/sql_parse.cc | 4 +- sql/sql_yacc.yy | 28 +- sql/sys_vars.inl | 2 + storage/innobase/CMakeLists.txt | 2 + storage/innobase/buf/buf0flu.cc | 99 +- storage/innobase/dict/dict0load.cc | 5 + storage/innobase/handler/backup_innodb.cc | 1113 +++++++++++++++++ storage/innobase/handler/backup_innodb.h | 58 + storage/innobase/handler/ha_innodb.cc | 54 +- storage/innobase/include/fil0fil.h | 49 + storage/innobase/include/log0log.h | 62 +- storage/innobase/log/log0log.cc | 78 +- storage/innobase/log/log0recv.cc | 4 +- storage/innobase/os/os0file.cc | 10 +- storage/maria/CMakeLists.txt | 1 + storage/maria/ha_maria.cc | 4 + storage/maria/ma_backup_server.cc | 423 +++++++ storage/maria/ma_backup_server.h | 58 + 93 files changed, 3710 insertions(+), 124 deletions(-) create mode 100644 mysql-test/main/backup_server.result create mode 100644 mysql-test/main/backup_server.test create mode 100644 mysql-test/main/backup_server_locking.result create mode 100644 mysql-test/main/backup_server_locking.test create mode 100644 mysql-test/main/grant_backup_server.result create mode 100644 mysql-test/main/grant_backup_server.test create mode 100644 mysql-test/suite/backup/backup_innodb,debug.rdiff create mode 100644 mysql-test/suite/backup/backup_innodb.combinations create mode 100644 mysql-test/suite/backup/backup_innodb.result create mode 100644 mysql-test/suite/backup/backup_innodb.test create mode 100644 mysql-test/suite/backup/backup_stream,debug.rdiff create mode 100644 mysql-test/suite/backup/backup_stream.combinations create mode 100644 mysql-test/suite/backup/backup_stream.result create mode 100644 mysql-test/suite/backup/backup_stream.test create mode 100644 mysql-test/suite/backup/suite.pm create mode 100644 sql/sql_backup.cc create mode 100644 sql/sql_backup.h create mode 100644 sql/sql_backup_interface.h create mode 100644 storage/innobase/handler/backup_innodb.cc create mode 100644 storage/innobase/handler/backup_innodb.h create mode 100644 storage/maria/ma_backup_server.cc create mode 100644 storage/maria/ma_backup_server.h diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index 80b202aa4c03e..59f386a86edae 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -166,6 +166,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/opt_hints.cc ../sql/opt_hints.h ../sql/opt_trace_ddl_info.cc ../sql/opt_trace_ddl_info.h ../sql/sql_path.cc + ../sql/sql_backup.cc ${GEN_SOURCES} ${MYSYS_LIBWRAP_SOURCE} ) diff --git a/mysql-test/collections/buildbot_suites.bat b/mysql-test/collections/buildbot_suites.bat index 053057872bf40..129a2f67d79de 100644 --- a/mysql-test/collections/buildbot_suites.bat +++ b/mysql-test/collections/buildbot_suites.bat @@ -6,6 +6,7 @@ innodb,^ versioning,^ plugins,^ mariabackup,^ +backup,^ roles,^ auth_gssapi,^ mysql_sha2,^ diff --git a/mysql-test/main/backup_server.result b/mysql-test/main/backup_server.result new file mode 100644 index 0000000000000..7a064d7e5fe11 --- /dev/null +++ b/mysql-test/main/backup_server.result @@ -0,0 +1,6 @@ +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; diff --git a/mysql-test/main/backup_server.test b/mysql-test/main/backup_server.test new file mode 100644 index 0000000000000..50cf326d39838 --- /dev/null +++ b/mysql-test/main/backup_server.test @@ -0,0 +1,10 @@ +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory diff --git a/mysql-test/main/backup_server_locking.result b/mysql-test/main/backup_server_locking.result new file mode 100644 index 0000000000000..6e49a09e3e873 --- /dev/null +++ b/mysql-test/main/backup_server_locking.result @@ -0,0 +1,17 @@ +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +BACKUP STAGE START; +connect backup,localhost,root; +SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR 70100: Query was interrupted: execution time limit 0.1 sec exceeded +connection default; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't execute the command as you have a BACKUP STAGE active +BACKUP STAGE END; +connection backup; +SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +disconnect backup; +connection default; diff --git a/mysql-test/main/backup_server_locking.test b/mysql-test/main/backup_server_locking.test new file mode 100644 index 0000000000000..4e769b61b2f20 --- /dev/null +++ b/mysql-test/main/backup_server_locking.test @@ -0,0 +1,31 @@ +--source include/not_embedded.inc +--source include/count_sessions.inc + +--mkdir $MYSQLTEST_VARDIR/some_directory +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +BACKUP STAGE START; +--connect (backup,localhost,root) +--error ER_STATEMENT_TIMEOUT +evalp SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +--connection default + +--error ER_BACKUP_LOCK_IS_ACTIVE +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +BACKUP STAGE END; +--connection backup +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--disconnect backup +--connection default + +--rmdir $MYSQLTEST_VARDIR/some_directory + +--source include/wait_until_count_sessions.inc diff --git a/mysql-test/main/grant_backup_server.result b/mysql-test/main/grant_backup_server.result new file mode 100644 index 0000000000000..a4d85292dee45 --- /dev/null +++ b/mysql-test/main/grant_backup_server.result @@ -0,0 +1,27 @@ +CREATE USER user1@localhost IDENTIFIED BY ''; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON test.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT RELOAD ON test.* TO user1@localhost; +ERROR HY000: Incorrect usage of DB GRANT and GLOBAL PRIVILEGES +GRANT RELOAD ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the SELECT privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +disconnect con1; +connection default; +DROP USER user1@localhost; diff --git a/mysql-test/main/grant_backup_server.test b/mysql-test/main/grant_backup_server.test new file mode 100644 index 0000000000000..726abb19908bb --- /dev/null +++ b/mysql-test/main/grant_backup_server.test @@ -0,0 +1,29 @@ +--source include/not_embedded.inc +CREATE USER user1@localhost IDENTIFIED BY ''; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON test.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +--error ER_WRONG_USAGE +GRANT RELOAD ON test.* TO user1@localhost; +GRANT RELOAD ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +--disconnect con1 +--connection default +DROP USER user1@localhost; diff --git a/mysql-test/main/mysqld--help.result b/mysql-test/main/mysqld--help.result index 623311c170616..e55576862a7c2 100644 --- a/mysql-test/main/mysqld--help.result +++ b/mysql-test/main/mysqld--help.result @@ -2040,7 +2040,7 @@ performance-schema-max-socket-classes 10 performance-schema-max-socket-instances -1 performance-schema-max-sql-text-length 1024 performance-schema-max-stage-classes 170 -performance-schema-max-statement-classes 227 +performance-schema-max-statement-classes 228 performance-schema-max-statement-stack 10 performance-schema-max-table-handles -1 performance-schema-max-table-instances -1 diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index 788927069015e..936e3c4d8fdf6 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -180,6 +180,7 @@ END main- archive- atomic- + backup- binlog- binlog_encryption- binlog_in_engine- diff --git a/mysql-test/suite/backup/backup_innodb,debug.rdiff b/mysql-test/suite/backup/backup_innodb,debug.rdiff new file mode 100644 index 0000000000000..592e1bf5e57ad --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb,debug.rdiff @@ -0,0 +1,16 @@ +--- backup_innodb.result ++++ backup_innodb,debug.result +@@ -20,7 +20,13 @@ + BEGIN; + DELETE FROM t; + connect backup,localhost,root; ++SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; + BACKUP SERVER TO 'target_directory' 4 CONCURRENT; ++connection default; ++SET DEBUG_SYNC='now WAIT_FOR start'; ++INSERT INTO t(a) SELECT * FROM seq_1_to_30000; ++SET DEBUG_SYNC='now SIGNAL resume'; ++connection backup; + disconnect backup; + connection default; + ROLLBACK; diff --git a/mysql-test/suite/backup/backup_innodb.combinations b/mysql-test/suite/backup/backup_innodb.combinations new file mode 100644 index 0000000000000..7fd419bba7692 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.combinations @@ -0,0 +1,4 @@ +[archived] +innodb_log_archive=ON +[circular] +innodb_log_archive=OFF diff --git a/mysql-test/suite/backup/backup_innodb.result b/mysql-test/suite/backup/backup_innodb.result new file mode 100644 index 0000000000000..bff88f510bfe9 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.result @@ -0,0 +1,60 @@ +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (2), (3), (5), (7); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (1), (1), (2), (3), (5); +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; +BACKUP SERVER TO '$target_directory'; +ROLLBACK; +SELECT * FROM t; +a b +1 +BACKUP SERVER TO '$target_directory' WITH '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH '/bin/false'' at line 1 +BACKUP SERVER TO '$target_directory' WITH 4 CONCURRENT '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH 4 CONCURRENT '/bin/false'' at line 1 +BACKUP SERVER WITH '/bin/false'; +ERROR HY000: IO Write error: (...) BACKUP SERVER +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +BACKUP SERVER TO 'target_directory' 4 CONCURRENT; +disconnect backup; +connection default; +ROLLBACK; +SELECT * FROM t; +a b +1 +DELETE FROM t; +DROP TABLE at0, at1; +# restart: --defaults-file=MYSQLTEST_VARDIR/some_directory/backup.cnf --datadir=MYSQLTEST_VARDIR/some_directory +SELECT * FROM t; +a b +1 +DELETE FROM t; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +i +1 +1 +2 +3 +5 +SELECT * FROM at1; +i +2 +3 +5 +7 +DROP TABLE t, at0, at1; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +ERROR 42S02: Table 'test.at0' doesn't exist +SELECT * FROM at1; +ERROR 42S02: Table 'test.at1' doesn't exist +# restart +SELECT * FROM t; +a b +DROP TABLE t; diff --git a/mysql-test/suite/backup/backup_innodb.test b/mysql-test/suite/backup/backup_innodb.test new file mode 100644 index 0000000000000..c58cff8eec21a --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.test @@ -0,0 +1,104 @@ +--source include/have_sequence.inc +--source include/have_innodb.inc +--source include/maybe_debug.inc + +--disable_query_log +call mtr.add_suppression("mariadbd.*: IO Write error:"); +--enable_query_log + +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (2), (3), (5), (7); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (1), (1), (2), (3), (5); + +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; + +--let $target_directory=/tmp/some_directory$MTR_COMBINATION_ARCHIVED +# comment out the following line (and replace all "rmdir" with "exec rm -fr") +# to test cross-filesystem copy +--let $target_directory=$MYSQLTEST_VARDIR/some_directory + +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory + +evalp BACKUP SERVER TO '$target_directory'; +--rmdir $target_directory +ROLLBACK; +# BACKUP SERVER will implicitly commit the current transaction +SELECT * FROM t; + +--error ER_PARSE_ERROR +evalp BACKUP SERVER TO '$target_directory' WITH '/bin/false'; +--error ER_PARSE_ERROR +evalp BACKUP SERVER TO '$target_directory' WITH 4 CONCURRENT '/bin/false'; +--replace_regex /\(.*\)/(...)/ +--error ER_IO_WRITE_ERROR +BACKUP SERVER WITH '/bin/false'; + +BEGIN; +DELETE FROM t; + +--connect backup,localhost,root +if ($have_debug) { +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; +--replace_result $target_directory target_directory +send_eval BACKUP SERVER TO '$target_directory' 4 CONCURRENT; +--connection default +SET DEBUG_SYNC='now WAIT_FOR start'; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +SET DEBUG_SYNC='now SIGNAL resume'; +--connection backup +--reap +} +if (!$have_debug) { +--replace_result $target_directory target_directory +eval BACKUP SERVER TO '$target_directory' 4 CONCURRENT; +} + +--disconnect backup +--connection default +ROLLBACK; +SELECT * FROM t; +DELETE FROM t; +DROP TABLE at0, at1; + +if ($MARIADB_UPGRADE_EXE) { +let target_directory=$target_directory; +perl; +open IN, "<", "$ENV{MYSQLTEST_VARDIR}/my.cnf"; +open OUT, ">>", "$ENV{target_directory}/backup.cnf"; +print OUT while (); +close(IN); +close(OUT); +EOF +} +if (!$MARIADB_UPGRADE_EXE) { + --exec cat $MYSQLTEST_VARDIR/my.cnf >> $target_directory/backup.cnf +} +--let $restart_parameters=--defaults-file=$target_directory/backup.cnf --datadir=$target_directory +--source include/restart_mysqld.inc + +SELECT * FROM t; +# A nonzero innodb_log_recovery_target makes InnoDB read-only. +--error ER_OPEN_AS_READONLY +DELETE FROM t; +# Non-InnoDB tables are read-write. +SELECT * FROM at0; +SELECT * FROM at1; +--error ER_OPEN_AS_READONLY +DROP TABLE t, at0, at1; +--error ER_NO_SUCH_TABLE +SELECT * FROM at0; +--error ER_NO_SUCH_TABLE +SELECT * FROM at1; + +--let $restart_parameters= +--source include/restart_mysqld.inc +SELECT * FROM t; +DROP TABLE t; + +--rmdir $target_directory diff --git a/mysql-test/suite/backup/backup_stream,debug.rdiff b/mysql-test/suite/backup/backup_stream,debug.rdiff new file mode 100644 index 0000000000000..8130727323dec --- /dev/null +++ b/mysql-test/suite/backup/backup_stream,debug.rdiff @@ -0,0 +1,16 @@ +--- backup_stream.result ++++ backup_stream,debug.result +@@ -8,7 +8,13 @@ + BEGIN; + DELETE FROM t; + connect backup,localhost,root; ++SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; + BACKUP SERVER WITH 2 CONCURRENT 'stream.bat'; ++connection default; ++SET DEBUG_SYNC='now WAIT_FOR start'; ++INSERT INTO t(a) SELECT * FROM seq_1_to_30000; ++SET DEBUG_SYNC='now SIGNAL resume'; ++connection backup; + disconnect backup; + connection default; + ROLLBACK; diff --git a/mysql-test/suite/backup/backup_stream.combinations b/mysql-test/suite/backup/backup_stream.combinations new file mode 100644 index 0000000000000..7fd419bba7692 --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.combinations @@ -0,0 +1,4 @@ +[archived] +innodb_log_archive=ON +[circular] +innodb_log_archive=OFF diff --git a/mysql-test/suite/backup/backup_stream.result b/mysql-test/suite/backup/backup_stream.result new file mode 100644 index 0000000000000..e12a2f683c286 --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.result @@ -0,0 +1,35 @@ +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +INSERT INTO t SET a=1; +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (1); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (0); +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +BACKUP SERVER WITH 2 CONCURRENT 'stream.bat'; +disconnect backup; +connection default; +ROLLBACK; +SELECT * FROM t; +a b +1 +DELETE FROM t; +DROP TABLE at0, at1; +# restart: --defaults-file=MYSQLTEST_VARDIR/backup_stream/backup.cnf --datadir=MYSQLTEST_VARDIR/backup_stream +SELECT * FROM at0; +i +0 +SELECT * FROM at1; +i +1 +SELECT * FROM t; +a b +1 +DELETE FROM t; +ERROR HY000: Table 't' is read only +# restart +SELECT * FROM t; +a b +DROP TABLE t; diff --git a/mysql-test/suite/backup/backup_stream.test b/mysql-test/suite/backup/backup_stream.test new file mode 100644 index 0000000000000..0ad21d49a663c --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.test @@ -0,0 +1,85 @@ +--source include/have_sequence.inc +--source include/have_innodb.inc +--source include/maybe_debug.inc + +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +INSERT INTO t SET a=1; + +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (1); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (0); + +--remove_files_wildcard $MYSQL_TMP_DIR/. stream.bat +--write_file $MYSQL_TMP_DIR/stream.bat +#!/bin/sh +exec cat > $MYSQL_TMP_DIR/$1.tar +EOF + +--let $script=$MYSQL_TMP_DIR/stream.bat +if (!$MARIADB_UPGRADE_EXE) { +# Because we may run on Linux /dev/shm which may be mounted as noexec, +# we cannot rely on chmod +x, but must explicitly invoke a shell on the script. +--let $script=/bin/sh $script +} +if ($MARIADB_UPGRADE_EXE) +{ +--exec echo "@cat > %MYSQL_TMP_DIR%\%1.tar" > $MYSQL_TMP_DIR/stream.bat +} + +BEGIN; +DELETE FROM t; + +--connect backup,localhost,root +if ($have_debug) { +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; +--replace_result $script stream.bat +send_eval BACKUP SERVER WITH 2 CONCURRENT '$script'; +--connection default +SET DEBUG_SYNC='now WAIT_FOR start'; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +SET DEBUG_SYNC='now SIGNAL resume'; +--connection backup +--reap +} +if (!$have_debug) { +--replace_result $script stream.bat +eval BACKUP SERVER WITH 2 CONCURRENT '$script'; +} + +--remove_file $MYSQL_TMP_DIR/stream.bat +--disconnect backup +--connection default +ROLLBACK; +SELECT * FROM t; +DELETE FROM t; +DROP TABLE at0, at1; + +--let $target_directory=$MYSQLTEST_VARDIR/backup_stream +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory +--mkdir $target_directory +--exec tar xf $MYSQL_TMP_DIR/1.tar -C $target_directory +--exec tar xf $MYSQL_TMP_DIR/2.tar -C $target_directory + +--exec cat $MYSQLTEST_VARDIR/my.cnf >> $target_directory/backup.cnf + +--let $restart_parameters=--defaults-file=$target_directory/backup.cnf --datadir=$target_directory +--source include/restart_mysqld.inc +SELECT * FROM at0; +SELECT * FROM at1; +SELECT * FROM t; +--error ER_OPEN_AS_READONLY +DELETE FROM t; + +--remove_file $MYSQL_TMP_DIR/1.tar +--remove_file $MYSQL_TMP_DIR/2.tar + +--let $restart_parameters= +--source include/restart_mysqld.inc +SELECT * FROM t; +DROP TABLE t; + +--rmdir $target_directory diff --git a/mysql-test/suite/backup/suite.pm b/mysql-test/suite/backup/suite.pm new file mode 100644 index 0000000000000..7d36a22655efa --- /dev/null +++ b/mysql-test/suite/backup/suite.pm @@ -0,0 +1,19 @@ +package My::Suite::Backup; + +@ISA = qw(My::Suite); +use My::Find; +use File::Basename; +use strict; + +return "Not run for embedded server" if $::opt_embedded_server; + +my $have_cat = index(`echo meowl|cat 2>&1`,"meowl") >= 0; +my $have_tar = `tar --version 2>&1` =~ /tar .*\d\.\d/; + +sub skip_combinations { + my %skip; + $skip{'backup_stream.test'} = 'needs cat,tar' unless $have_cat && $have_tar; + %skip; +} + +bless { }; diff --git a/mysql-test/suite/perfschema/r/max_program_zero.result b/mysql-test/suite/perfschema/r/max_program_zero.result index 047643e06988d..a0e486b2af9c0 100644 --- a/mysql-test/suite/perfschema/r/max_program_zero.result +++ b/mysql-test/suite/perfschema/r/max_program_zero.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 1 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/ortho_iter.result b/mysql-test/suite/perfschema/r/ortho_iter.result index 56c22c8453d8e..589704f4056de 100644 --- a/mysql-test/suite/perfschema/r/ortho_iter.result +++ b/mysql-test/suite/perfschema/r/ortho_iter.result @@ -251,7 +251,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/privilege_table_io.result b/mysql-test/suite/perfschema/r/privilege_table_io.result index 0c82be9f05810..b518052613308 100644 --- a/mysql-test/suite/perfschema/r/privilege_table_io.result +++ b/mysql-test/suite/perfschema/r/privilege_table_io.result @@ -57,7 +57,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_idle.result b/mysql-test/suite/perfschema/r/start_server_disable_idle.result index d0665e3bf4c65..12b956d90769c 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_idle.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_idle.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_stages.result b/mysql-test/suite/perfschema/r/start_server_disable_stages.result index 2ef68328144ff..30ce9d12e56a0 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_stages.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_stages.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_statements.result b/mysql-test/suite/perfschema/r/start_server_disable_statements.result index 0ece2a0c52ed1..4bacdbdfedf68 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_statements.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_statements.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result index ededc09aac95d..3a6831eb2c7ff 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_waits.result b/mysql-test/suite/perfschema/r/start_server_disable_waits.result index 23db9362161e4..a864576dfa979 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_waits.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_waits.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_innodb.result b/mysql-test/suite/perfschema/r/start_server_innodb.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_innodb.result +++ b/mysql-test/suite/perfschema/r/start_server_innodb.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_index.result b/mysql-test/suite/perfschema/r/start_server_low_index.result index 11cade0a2132f..dbd36d2eaa92d 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_index.result +++ b/mysql-test/suite/perfschema/r/start_server_low_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result index 484550095202e..fab64f49d45e6 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_account.result b/mysql-test/suite/perfschema/r/start_server_no_account.result index aab8d3eba9caa..e2cccdba19b43 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_account.result +++ b/mysql-test/suite/perfschema/r/start_server_no_account.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result index 4dfdab9de9f30..44b114013ace2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result index a8d0cbeca3855..27ecb59a40f17 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_class.result b/mysql-test/suite/perfschema/r/start_server_no_file_class.result index fcc01880a7107..5189d36618b05 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result index c56201e7d0a80..533c02383c7b8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_host.result b/mysql-test/suite/perfschema/r/start_server_no_host.result index 662beb3b88a49..e328ea3d3c26a 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_host.result +++ b/mysql-test/suite/perfschema/r/start_server_no_host.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_index.result b/mysql-test/suite/perfschema/r/start_server_no_index.result index ccff0cb113faa..686f430cdc440 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_index.result +++ b/mysql-test/suite/perfschema/r/start_server_no_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mdl.result b/mysql-test/suite/perfschema/r/start_server_no_mdl.result index ebe64409deb0c..c2b6dc3d9eb74 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result index 01a217c534bfd..2a4cdcf8ddd37 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result index 1b3efda5210a9..9b77c7b897c47 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result index 599498915f3f1..3a62653efc18f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result index 73ac1acb9f145..7dd5842809135 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result +++ b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result index 04aa037b960e3..3598c722b9453 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result index e8156711eb3f2..7a60805d2d03f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result index 2d17bd6f49203..76da2883ba6b2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result index 16afee28ee70b..58ac763394a96 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result index 9de0006aa7abc..5a86d51a06231 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 0 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result index bcef0e5c01b05..75ab84f4b3cb8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 0 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result index 1dda39dc79e92..bb750ebd34a26 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 0 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result index 95584521eceb2..6cf3f740fb1be 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result index da6c54b6bbafa..1f2716410a9bc 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result index 09a9a544f5b13..f1078542f69ef 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result index 5ce9874799e8b..a6fc523df2c60 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result index a170fe097fd23..304732cb37927 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 0 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result index 3f009de021d1a..3ef43495d0f22 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 0 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result index db4fe3413106b..20cf1a531ca51 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result index b1b6e614c43ac..e8eb4589fce56 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result index e540f3ce78cbd..4dbf292fc3c10 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result index 80da238ba0ed7..09168c68f9fe8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result index f5d32f0100968..9ec63e991deda 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_user.result b/mysql-test/suite/perfschema/r/start_server_no_user.result index cb249b4e242d3..861b7d897c9c5 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_user.result +++ b/mysql-test/suite/perfschema/r/start_server_no_user.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result index c419aad2995f3..710cef232da58 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result index 0b2bf39ff874f..df4351a8f358f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_off.result b/mysql-test/suite/perfschema/r/start_server_off.result index 17db0395d894b..f3bf7bf7fb15e 100644 --- a/mysql-test/suite/perfschema/r/start_server_off.result +++ b/mysql-test/suite/perfschema/r/start_server_off.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_on.result b/mysql-test/suite/perfschema/r/start_server_on.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_on.result +++ b/mysql-test/suite/perfschema/r/start_server_on.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_variables.result b/mysql-test/suite/perfschema/r/start_server_variables.result index f25f9ab69d1c3..b86116e099cdd 100644 --- a/mysql-test/suite/perfschema/r/start_server_variables.result +++ b/mysql-test/suite/perfschema/r/start_server_variables.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result index 442e212557b55..3e8469b881801 100644 --- a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result +++ b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 2 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 97fd9a7c0ef61..f0834ea3a1e12 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1064,7 +1064,7 @@ NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 18446744073709551615 NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL -READ_ONLY YES +READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_LOG_RECOVERY_TARGET SESSION_VALUE NULL diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index e54e894e1d0fc..fd16d901d64fb 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -163,6 +163,7 @@ SET (SQL_SOURCE grant.cc sql_explain.cc sql_analyze_stmt.cc + sql_backup.cc sql_join_cache.cc create_options.cc multi_range_read.cc opt_histogram_json.cc diff --git a/sql/handler.h b/sql/handler.h index 3ab9e0bcd1a8e..ab4026cf57600 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -1496,6 +1496,61 @@ struct transaction_participant ulonglong (*prepare_commit_versioned)(THD *thd, ulonglong *trx_id); }; +/** BACKUP SERVER target */ +struct backup_target +{ +#ifdef _WIN32 + /** Target directory path name, or nullptr if streaming */ + const char *path; +#else + /** Target directory descriptor, or -1 if streaming */ + int fd; +#endif +}; + +/** BACKUP SERVER worker specific context */ +struct backup_sink +{ +#ifdef _WIN32 + /** A value indicating an invalid stream */ + static constexpr HANDLE NO_STREAM{INVALID_HANDLE_VALUE}; + /** Target pipe, or NO_STREAM if path!=nullptr */ + HANDLE stream; +#else + /** A value indicating an invalid file descriptor or stream */ + static constexpr int NO_STREAM{-1}; + /** Target pipe, or NO_STREAM if copying to a directory */ + int stream; +#endif + /** storage engine context returned by handlerton::backup_start() */ + void *ha_data; +}; + +/** BACKUP SERVER execution phase */ +enum backup_phase +{ + /** finish backup, possibly after BACKUP_PHASE_ABORT */ + BACKUP_PHASE_FINISH= -2, + /** abort any operation */ + BACKUP_PHASE_ABORT= -1, + /** preparatory phase executed while holding no locks */ + BACKUP_PHASE_PREPARE_START= 0, + /** initial actual work phase; @see MDL_BACKUP_START */ + BACKUP_PHASE_START, + /** copy while new writes to non-transactional tables are blocked; + @see MDL_BACKUP_FLUSH */ + BACKUP_PHASE_NO_BEGIN_NON_TRANS, + /** copy while any writes to non-transactional tables are blocked; + @see MDL_BACKUP_WAIT_FLUSH */ + BACKUP_PHASE_NO_DML_NON_TRANS, + /** copy files while DDL is blocked; @see MDL_BACKUP_WAIT_DDL */ + BACKUP_PHASE_NO_DDL, + /** determine the logical time of the backup and copy any + remaining files while MDL_BACKUP_WAIT_COMMIT is active; + this is followed by BACKUP_PHASE_FINISH */ + BACKUP_PHASE_NO_COMMIT +}; + /* handlerton is a singleton structure - one instance per storage engine - to provide access to storage engine functionality that works on the @@ -1892,9 +1947,48 @@ struct handlerton : public transaction_participant /********************************************************************* backup **********************************************************************/ + + /** BACKUP STAGE START */ void (*prepare_for_backup)(void); + /** BACKUP STAGE END */ void (*end_backup)(void); + /** + Start of a BACKUP SERVER phase, + when no backup_step() or backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_PREPARE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to sink, or nullptr + @retval -1 on failure + */ + void *(*backup_start)(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink); + /** + Process a file that was collected in backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int (*backup_step)(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink); + /** + Finish a phase, once all calls for the current phase are completed. + @param thd current sesssion + @param target backup target + @param phase last phase on which backup_start() was successfully + invoked, or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success + */ + int (*backup_end)(THD *thd, const backup_target *target, backup_phase phase, + const backup_sink *sink); + /********************************************************************** WSREP specific **********************************************************************/ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index e5358a3a037eb..b178f32362925 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3537,6 +3537,7 @@ SHOW_VAR com_status_vars[]= { {"assign_to_keycache", STMT_STATUS(SQLCOM_ASSIGN_TO_KEYCACHE)}, {"backup", STMT_STATUS(SQLCOM_BACKUP)}, {"backup_lock", STMT_STATUS(SQLCOM_BACKUP_LOCK)}, + {"backup_server", STMT_STATUS(SQLCOM_BACKUP_SERVER)}, {"begin", STMT_STATUS(SQLCOM_BEGIN)}, {"binlog", STMT_STATUS(SQLCOM_BINLOG_BASE64_EVENT)}, {"call_procedure", STMT_STATUS(SQLCOM_CALL)}, diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc new file mode 100644 index 0000000000000..f4ad618b4694a --- /dev/null +++ b/sql/sql_backup.cc @@ -0,0 +1,886 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "mdl.h" +#include "mysys_err.h" +#include "sql_class.h" +#include "sql_backup.h" +#include "sql_backup_interface.h" +#include "sql_parse.h" +#include "my_atomic_wrapper.h" +#include "tpool.h" +#include "aligned.h" + +#if defined __linux__ || defined __FreeBSD__ +using copying_step= ssize_t(int,int,size_t,off_t*); +template +static ssize_t copy(int in_fd, int out_fd, off_t offset, off_t end) noexcept +{ + for (;;) + { + const size_t c{size_t(std::min(end - offset, INT_MAX >> 20 << 20))}; + ssize_t ret= step(in_fd, out_fd, c, &offset); + if (ret < 0) + { + if (nonblocking && errno == EAGAIN) + continue; + return ret; + } + if (offset == end) + return 0; + if (!ret) + return -1; + } +} + +# if 1 // disable to work around https://github.com/rr-debugger/rr/issues/4059 +/* Copy between files in a single (type of) file system */ +static inline ssize_t +copy_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return copy_file_range(in_fd, offset, out_fd, offset, count, 0); +} +# define cfr(src,dst,start,end) copy(src, dst, start, end) +# endif +#endif + +#ifdef _WIN32 +using tpool::pread; +using tpool::pwrite; +#else +# include +/** + Copy a file using a memory mapping. + @tparam stream true=write to a stream, false=pwrite to a file + @param in_fd source file + @param out_fd destination + @param o start offset + @param end last offset (exclusive) + @return error code + @retval 0 on success + @retval 1 if a memory mapping failed +*/ +template +static ssize_t mmap_copy(int in_fd, int out_fd, uint64_t o, uint64_t end) +{ +# if SIZEOF_SIZE_T < 8 + if (end != size_t(end)) + return 1; +# endif + const size_t count= size_t(end - o); + void *p= mmap(nullptr, count, PROT_READ, MAP_SHARED, in_fd, off_t(o)); + if (p == MAP_FAILED) + return 1; + ssize_t ret; + size_t c{count}; + for (const char *b= static_cast(p);; b+= ret, o+= uint64_t(ret)) + { + const size_t size{std::min(c, size_t(INT_MAX >> 20 << 20))}; + if (stream) + if ((ret= backup_stream_write(out_fd, b, size))) + break; + ret= stream ? size : pwrite(out_fd, b, size, off_t(o)); + if (ret < 0) + break; + c-= ret; + if (!c) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + munmap(p, c); + return ret; +} +#endif + +/** + Copy a file using positioned reads. + @tparam stream true=write to a stream, false=pwrite to a file + @param in_fd source file + @param out_fd destination + @param o start offset + @param end last offset (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +template +static ssize_t pread_write(IF_WIN(const native_file_handle&,int) in_fd, + IF_WIN(const native_file_handle&,int) out_fd, + uint64_t o, uint64_t end) + noexcept +{ + constexpr size_t READ_WRITE_SIZE= 65536; + char *b= static_cast(aligned_malloc(READ_WRITE_SIZE, 4096)); + if (!b) + return -1; + ssize_t ret; + for (uint64_t count{end - o};; o+= ret) + { + ret= pread(in_fd, b, + ssize_t(std::min(count, READ_WRITE_SIZE)), o); + if (ret > 0) + { + if (!stream) + ret= pwrite(out_fd, b, size_t(ret), o); + else if (backup_stream_write(out_fd, b, size_t(ret))) + { + ret= -1; + break; + } + } + if (ret < 0) + break; + count-= uint64_t(ret); + if (!count) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + aligned_free(b); + return ret; +} + +#ifdef __APPLE__ +/* The inline copy_entire_file() invokes fcopyfile() */ +#elif defined _WIN32 +/* CopyFileEx() should be used */ +#else +/** Copy a file (whole content). +@param src source file descriptor +@param dst target to append src to +@return error code (non-positive) +@retval 0 on success */ +extern "C" int copy_entire_file(int src, int dst) +{ + return copy_file(src, dst, 0, lseek(src, 0, SEEK_END)); +} +#endif + +/** Copy a portion of a file. +@param src source file descriptor +@param dst target to append src to +@param start first offset to copy +@param end last offset to copy (exclusive) +@return error code (non-positive) +@retval 0 on success */ +extern "C" int copy_file(IF_WIN(const native_file_handle&,int) src, + IF_WIN(const native_file_handle&,int) dst, + uint64_t start, uint64_t end) +{ + assert(end >= start); + ssize_t ret; +# ifdef cfr + if (!(ret= cfr(src, dst, off_t(start), off_t(end)))) + return int(ret); +# ifdef __linux__ + if (errno == EOPNOTSUPP || errno == EXDEV) +# endif +# endif +# ifdef __linux__ // starting with Linux 2.6.33, we can rely on sendfile(2) + return (start != 0 && off_t(start) != lseek(dst, start, SEEK_SET)) + ? -1 + : backup_stream_append_async(src, dst, start, end); +# else +# ifndef _WIN32 + if ((ret= mmap_copy(src, dst, start, end)) == 1) +# endif + ret= pread_write(src, dst, start, end); +# endif + assert(ret <= 0); + return int(ret); +} + +/** Append to the configuration file. +@param target backup target directory +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +extern "C" int backup_config_append(IF_WIN(const char*, int) target, + const char *config, size_t size) +{ + /* FIXME: append to a pre-created configuration file */ +#ifdef _WIN32 + HANDLE dst; + { + std::string path{target}; + path.append("/backup.cnf"); + dst= CreateFile(path.c_str(), GENERIC_WRITE, 0, + my_win_file_secattr(), CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (dst != INVALID_HANDLE_VALUE) + { + BOOL ok; + for (;;) + { + DWORD written; + ok= WriteFile(dst, config, DWORD(size), &written, nullptr); + if (ok || GetLastError() != ERROR_IO_PENDING) + break; + assert(written < DWORD(size)); + config+= written; + size-= size_t(written); + } + if (CloseHandle(dst) & ok) + return 0; + } + } +#else + int dst= openat(target, "backup.cnf", + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (dst < 0) + return dst; + ssize_t ret; + for (; (ret= write(dst, config, size)) >= 0; config+= ret, size -= ret) + { + assert(size_t(ret) <= size); + if (!(size-= size_t(ret))) + { + ret= 0; + break; + } + } + if (!(close(dst) | ret)) + return 0; +#endif + my_error(ER_CANT_CREATE_FILE, MYF(0), "backup.cnf", errno); + return -1; +} + +/** backup context */ +struct backup_target_phase +{ + /** target directory or stream */ + backup_target target; + /** current phase of backup */ + backup_phase phase; + /** backup worker state (output stream) */ + backup_sink sink; + /** stream object for streaming backup */ + FILE *stream; + /** handlerton::backup_step return value in multi-threaded operation */ + int ret; + /** engine-specific backup context */ + std::unordered_map &ha_data; +}; + +/** + Inform a storage engine of an upcoming backup by invoking + handlerton::backup_start(BACKUP_PHASE_PREPARE_START) before + acquiring any locks. + @param thd current session + @param plugin storage engine + @return whether the operation failed +*/ +static my_bool backup_preparation(THD *thd, plugin_ref plugin, void*) noexcept +{ + const auto bs= plugin_hton(plugin)->backup_start; + return bs && bs(thd, nullptr, BACKUP_PHASE_PREPARE_START, nullptr); +} + +/** + Invoke handlerton::backup_start() on a storage engine, + when there are no pending handlerton::backup_step() in any thread. + @param thd current session + @param plugin storage engine + @param arg the backup_target_phase context + @return whether the operation failed +*/ +static my_bool backup_start(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + assert(int{t.phase} >= 0 || t.phase == BACKUP_PHASE_FINISH); + if (hton->backup_start) + { + t.sink.ha_data= t.ha_data[hton]; + void *data= hton->backup_start(thd, &t.target, t.phase, &t.sink); + if (data == reinterpret_cast(-1)) + return true; + assert(!t.ha_data[hton] || t.ha_data[hton] == data); + t.ha_data[hton]= data; + } + return false; +} + +/** + Invoke handlerton::backup_end() on a storage engine, + when there are no pending handlerton::backup_step() in any thread. + @param thd current session + @param plugin storage engine + @param arg the backup_target_phase context + @return whether the operation failed +*/ +static my_bool backup_end(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + if (hton->backup_end) + { + t.sink.ha_data= t.ha_data[hton]; + return hton->backup_end(thd, &t.target, t.phase, &t.sink); + } + return false; +} + +/** + Invoke handlerton::backup_step() on a storage engine in a thread + that may or may not be associated with a BACKUP SERVER connection, + between handlerton::backup_start() and handlerton::backup_end() + of the same backup_phase. + @param thd the BACKUP SERVER session + @param plugin storage engine + @param arg the backup_target_phase context + @return whether the operation failed +*/ +static my_bool backup_step(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + assert(int{t.phase} >= 0 || t.phase == BACKUP_PHASE_FINISH); + int res= 0; + if (hton->backup_step) + { + t.sink.ha_data= t.ha_data[hton]; + while ((res= hton->backup_step(thd, &t.target, t.phase, &t.sink))) + if (res < 0) + break; + } + return res != 0; +} + +/** Number of background tasks executing backup_step_callback */ +static Atomic_counter backup_step_callback_pending{0}; + +/** Invoke backup_step() in a background task */ +static void backup_step_callback(void *arg) noexcept +{ + backup_target_phase &t{*static_cast(arg)}; + assert(!t.ret); + t.ret= plugin_foreach_with_mask(nullptr, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, &t); +#ifndef NDEBUG + auto was_pending= +#endif + backup_step_callback_pending--; + assert(was_pending); +} + +/** + Execute all handlerton::backup_step() until completion or failure. + @param thd current connection + @param target_phase backup target and phase + @param threads number of execution threads + @param tp thread pool +*/ +static bool backup_steps(THD *thd, backup_target_phase *target_phase, + int threads, tpool::thread_pool *tp) +{ + assert(!backup_step_callback_pending); + if (threads == 1) + return plugin_foreach_with_mask(thd, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + tpool::task *const tasks= + static_cast(alloca(threads * sizeof *tasks)); + backup_step_callback_pending= threads - 1; + for (int n{threads}; --n; ) + { + target_phase[n].phase= target_phase->phase; + tp->submit_task(new (&tasks[n]) tpool::task{backup_step_callback, + &target_phase[n]}); + } + bool fail= plugin_foreach_with_mask(thd, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + while (backup_step_callback_pending) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + if (fail) + return fail; + + for (int n{threads}; --n; ) + if (target_phase[n].ret) + { + my_error(ER_UNKNOWN_ERROR, MYF(0)); + return true; + } + + return false; +} + +bool Sql_cmd_backup::execute(THD *thd) +{ + assert(!!target == !command); + + if (check_global_access(thd, RELOAD_ACL) || + check_global_access(thd, SELECT_ACL) || + (target && error_if_data_home_dir(target, "BACKUP SERVER TO"))) + return true; + + if (thd->current_backup_stage != BACKUP_FINISHED) + { + my_error(ER_BACKUP_LOCK_IS_ACTIVE, MYF(0)); + return true; + } + + bool fail{plugin_foreach_with_mask(thd, backup_preparation, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + nullptr)}; + if (fail) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return true; + } + + /* Block concurrent BACKUP SERVER and BACKUP STAGE */ + MDL_request mdl_request; + MDL_REQUEST_INIT(&mdl_request, MDL_key::BACKUP, "", "", MDL_BACKUP_START, + MDL_EXPLICIT); + + if (thd->mdl_context.acquire_lock(&mdl_request, + thd->variables.lock_wait_timeout)) + return true; + + tpool::thread_pool *tp= nullptr; + std::unordered_map ha_data{}; + backup_target_phase *target_phase= static_cast + (alloca(threads * sizeof *target_phase)); + if (threads > 1 && !(tp= tpool::create_thread_pool_generic())) + { + oor: + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + err_exit: + thd->mdl_context.release_lock(mdl_request.ticket); + delete tp; + return true; + } + + if (command) + { + char cmd[1024]; + for (int t{threads}; t; ) + { + if (snprintf(cmd, sizeof cmd, "%s %d", command, t) >= int(sizeof cmd)) + goto oor; + FILE *f= my_popen(cmd, "w"); + if (!f) + { + while (t < threads) + my_pclose(target_phase[t++].stream); + goto oor; + } +#ifdef _WIN32 + HANDLE sink= (HANDLE) _get_osfhandle(_fileno(f)); +#else + int sink= fileno(f); +#endif + new (&target_phase[--t]) + backup_target_phase{backup_target{IF_WIN(nullptr, -1)}, + BACKUP_PHASE_START, backup_sink{sink, nullptr}, f, 0, ha_data}; + } + } + else if (my_mkdir(target, 0755, MYF(MY_WME))) + goto err_exit; + else + { +#ifndef _WIN32 + const int dir{open(target, O_DIRECTORY)}; + if (dir < 0) + { + my_error(EE_CANT_MKDIR, MYF(ME_BELL), target, errno); + goto err_exit; + } +#endif + for (int t{threads}; t; ) + { + new (&target_phase[--t]) + backup_target_phase{backup_target{IF_WIN(target, dir)}, + BACKUP_PHASE_START, + backup_sink{backup_sink::NO_STREAM, nullptr}, nullptr, 0, ha_data}; + } + } + + static_assert(int{MDL_BACKUP_START} + 1 == int{MDL_BACKUP_FLUSH}, ""); + static_assert(int{MDL_BACKUP_START} + 2 == int{MDL_BACKUP_WAIT_FLUSH}, ""); + static_assert(int{MDL_BACKUP_START} + 3 == int{MDL_BACKUP_WAIT_DDL}, ""); + static_assert(int{MDL_BACKUP_START} + 4 == int{MDL_BACKUP_WAIT_COMMIT}, ""); + static_assert(int{BACKUP_PHASE_START} + 1 == + int{BACKUP_PHASE_NO_BEGIN_NON_TRANS}, ""); + static_assert(int{BACKUP_PHASE_START} + 2 == + int{BACKUP_PHASE_NO_DML_NON_TRANS}, ""); + static_assert(int{BACKUP_PHASE_START} + 3 == int{BACKUP_PHASE_NO_DDL}, ""); + static_assert(int{BACKUP_PHASE_START} + 4 == + int{BACKUP_PHASE_NO_COMMIT}, ""); + int phase= int{BACKUP_PHASE_START}; + goto backup_phase_start; + + for (; phase <= int{BACKUP_PHASE_NO_COMMIT}; phase++) + { + assert(!fail); + { + const enum_mdl_type mdl= + enum_mdl_type(phase - int{BACKUP_PHASE_START} + int{MDL_BACKUP_START}); + fail= + thd->mdl_context.upgrade_shared_lock(mdl_request.ticket, mdl, + thd->variables.lock_wait_timeout); + if (fail) + break; + } + backup_phase_start: + target_phase->phase= backup_phase(phase); + fail= plugin_foreach_with_mask(thd, backup_start, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + if (fail) + break; + fail= backup_steps(thd, target_phase, threads, tp) || + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + if (fail) + break; + } + + /* The final part must not interfere with the use of the server datadir. + Release the locks. */ + thd->mdl_context.release_lock(mdl_request.ticket); + if (!fail) + { + target_phase->phase= BACKUP_PHASE_FINISH; + fail= plugin_foreach_with_mask(thd, backup_start, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase) || + backup_steps(thd, target_phase, threads, tp); + } + else + { + target_phase->phase= BACKUP_PHASE_ABORT; + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, target_phase); + target_phase->phase= BACKUP_PHASE_FINISH; + } + + fail= + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase) || fail; + delete tp; + + if (command) + for (int t= threads; t--; ) + my_pclose(target_phase[t].stream); +#ifndef _WIN32 + else + std::ignore= close(target_phase->target.fd); +#endif + + if (!fail) + my_ok(thd); + return fail; +} + +/** + Encode an octal string. + @param start first byte of buffer + @param end first byte after buffer + @param n number to encode +*/ +static void ustar_write_octal(char *start, char *end, uint64_t n) noexcept +{ + for (*--end= '\0'; --end >= start; n>>= 3) + *end= char('0' + (n & 7)); +} + +/** + Encode a quantity in 12 bytes. + @param start first byte of the buffer + @param n number to encode +*/ +static void ustar_write_dozen(char *start, uint64_t n) noexcept +{ + if (n < 1ULL << 33) + ustar_write_octal(start, start + 12, n); + else + { + const uint32_t head{my_htobe32(1U << 31)}; + n= my_htobe64(n); + memcpy(start, &head, 4); + memcpy(start + 4, &n, 8); + } +} + +/** Initialize a ustar block +@param buf GNU tape archiver --format=oldgnu header block +@param name name of the block +@param mode file access mode +@param size physical size of the following block */ +static void ustar_block_init(char *buf, const char *name, mode_t mode, + uint64_t size) noexcept +{ + strncpy(buf, name, 100); + ustar_write_octal(buf + 100, buf + 108, uint64_t(mode)); + ustar_write_octal(buf + 108, buf + 116, 0/* POSIX uid */); + ustar_write_octal(buf + 116, buf + 124, 0/* POSIX gid */); + ustar_write_dozen(buf + 124, size); + /* last modification time */ + ustar_write_octal(buf + 136, buf + 148, 0); + memset(buf + 148, ' ', 9); /* initial block checksum and dummy type */ + memset(buf + 157, '\0', 100); /* name of linked file (unused) */ + memcpy(buf + 257, "ustar ", 8); + strncpy(buf + 265, "root" /* POSIX owner name */, 32); + strncpy(buf + 297, "root" /* POSIX group name */, 512 - 297); + /* The caller will fill in the rest and invoke ustar_block_checksum() */ +} + +/** + Compute and write the POSIX tar block checksum. + @param buf POSIX tar block +*/ +static void ustar_block_checksum(char *buf) noexcept +{ + uint16_t sum{0}; + for (int i{0}; i < 512; i++) + sum+= uint16_t{uint8_t(buf[i])}; + ustar_write_octal(buf + 148, buf + 155, sum); +} + +/** + Write data to a stream. + @param stream backup stream + @param buf source buffer + @param size length of the buffer (usually an integer multiple of 512) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_write(IF_WIN(HANDLE, int) stream, const void *buf, + size_t size) +{ +#ifdef _WIN32 + for (DWORD sz= DWORD(size);;) + { + DWORD wrote; + if (WriteFile(stream, buf, sz, &wrote, nullptr)) + { + assert(wrote == sz); + return 0; + } + else if (GetLastError() != ERROR_IO_PENDING) + return -1; + buf= static_cast(buf) + wrote; + sz-= wrote; + } +#else + do + { + ssize_t wrote= write(stream, buf, size); + assert(wrote <= ssize_t(size)); + if (wrote < 0) + { + if (errno == EAGAIN) + continue; + return -1; + } + buf= static_cast(buf) + wrote; + size-= size_t(wrote); + } + while (size); +#endif + return 0; +} + +/** + Copy a prefix of a NUL terminated string to a buffer, NUL-padded. + @param b output buffer + @param s NUL terminated string + @param size size of buf, in bytes +*/ +static inline char *ustar_zeropad(char *b, const char *s, size_t size) noexcept +{ +#if defined __GNUC__ && __GNUC__ >= 8 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wstringop-truncation" +#endif + return strncpy(b, s, size); +#if defined __GNUC__ && __GNUC__ >= 8 +# pragma GCC diagnostic pop +#endif +} + +/** Start streaming a file. +@param stream backup stream +@param name file name +@param mode file access mode +@param size physical length of the file, in bytes +@param chunks payload chunks of a sparse file, or nullptr +@param n_chunks number of chunks; 0 unless sparse file +@return error code (non-positive) +@retval 0 on success */ +extern "C" +int backup_stream_start(IF_WIN(HANDLE, int) stream, + const char *name, mode_t mode, uint64_t size, + const struct backup_chunk *chunks, size_t n_chunks) +{ + assert(stream != backup_sink::NO_STREAM); + char buf[512]; + size_t s= strlen(name); + if (s > 100) + { + /* Write a block that contains the full name length, + followed by blocks that contain the full name, in + tar --format=oldgnu */ + ustar_block_init(buf, "././@LongLink", 0644, s); + ustar_block_checksum(buf); + if (int err= backup_stream_write(stream, buf, sizeof buf)) + return err; + const size_t whole{s & ~(sizeof buf)}; + if (whole) + if (int err= backup_stream_write(stream, name, whole)) + return err; + if (s - whole) + { + ustar_zeropad(buf, name + whole, sizeof buf); + if (int err= backup_stream_write(stream, buf, sizeof buf)) + return err; + } + } + + ustar_block_init(buf, name, mode, size); + if (!n_chunks) + buf[156]= '0'; + else + { + buf[156]= 'S'; + char *h= &buf[386]; + if (n_chunks > 4) + return -1; // FIXME; support more chunks + for (size_t i= 0; i < n_chunks; i++, h+= 24) + { + ustar_write_dozen(h, chunks[i].offset); + ustar_write_dozen(h + 12, chunks[i].length); + } + ustar_write_dozen(&buf[0x1e3], chunks[n_chunks - 1].offset + + chunks[n_chunks - 1].length); + } + ustar_block_checksum(buf); + return backup_stream_write(stream, buf, sizeof buf); +} + +/** Append to the configuration file. +@param target backup stream +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +extern "C" int backup_stream_config(IF_WIN(HANDLE, int) stream, + const char *config, size_t size) +{ + /* FIXME: append to a pre-created configuration file */ + if (int ret= + backup_stream_start(stream, "backup.cnf", 0644, size, nullptr, 0)) + return ret; + char buf[512]; + const size_t whole{size & ~((sizeof buf) - 1)}; + if (whole) + if (int err= backup_stream_write(stream, config, whole)) + return err; + if (size == whole) + return 0; + ustar_zeropad(buf, config + whole, sizeof buf); + return backup_stream_write(stream, buf, sizeof buf); +} + +/** + Append a file snippet to stream, + after a corresponding call to backup_stream_start(). + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_append(IF_WIN(const native_file_handle&,int) src, + IF_WIN(HANDLE, int) stream, + uint64_t start, uint64_t end) +{ + assert(stream != backup_sink::NO_STREAM); + ssize_t ret; +#ifndef _WIN32 + if ((ret= mmap_copy(src, stream, start, end)) == 1) +#endif + ret= pread_write(src, stream, start, end); + return int(ret); +} + +#ifdef __linux__ +# include +/** Copy a file to a stream or to a regular file. */ +static inline ssize_t +send_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return sendfile(out_fd, in_fd, offset, count); +} + +/** + Append an immutable snippet of a file to the stream, + allowing Linux sendfile(2) to be invoked. + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_append_async(int src, int stream, + uint64_t start, uint64_t end) +{ + assert(stream != backup_sink::NO_STREAM); + return int(copy(src, stream, off_t(start), off_t(end))); +} +#endif + +#ifdef _WIN32 +extern "C" int backup_stream_append_plain(HANDLE src, HANDLE stream, + uint64_t start, uint64_t end) +{ + return backup_stream_append(src, stream, start, end); +} +#endif diff --git a/sql/sql_backup.h b/sql/sql_backup.h new file mode 100644 index 0000000000000..2759e0d64b43d --- /dev/null +++ b/sql/sql_backup.h @@ -0,0 +1,53 @@ +/***************************************************************************** +Copyright (c) 2026 MariaDB plc. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +#pragma once + +/** BACKUP SERVER */ +class Sql_cmd_backup : public Sql_cmd +{ + /** target directory, or nullptr when streaming */ + const char *const target{nullptr}; + /** argument of my_popen() for streaming backup, or nullptr */ + const char *const command{nullptr}; + /** number of concurrent threads to use */ + const int threads; + +public: + /** + Constructor. + @param target name of target or scratch directory + @param threads number of concurrent threads to use + */ + Sql_cmd_backup(LEX_CSTRING target, int threads) : + target(target.str), threads(threads) {} + /** + Constructor. + @param threads number of concurrent threads to use + @param command nullptr, or a shell command for handling a backup stream + */ + Sql_cmd_backup(int threads, LEX_CSTRING command) : + command(command.str), threads(threads) {} + ~Sql_cmd_backup()= default; + + bool execute(THD *thd) override; + + enum_sql_command sql_command_code() const override + { + return SQLCOM_BACKUP_SERVER; + } +}; diff --git a/sql/sql_backup_interface.h b/sql/sql_backup_interface.h new file mode 100644 index 0000000000000..94892ba7591ab --- /dev/null +++ b/sql/sql_backup_interface.h @@ -0,0 +1,178 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +struct backup_target; + +/** A payload chunk in a sparse file that is being streamed */ +struct backup_chunk +{ + /** byte offset of the start of the payload, from the start of the file */ + uint64_t offset; + /** length of the hole */ + uint64_t length; +}; + +#ifdef _WIN32 +/* Use CopyFileEx() to copy entire files */ +struct native_file_handle; +#elif defined __APPLE__ +/* You should invoke fclonefileat(2) manually before attempting +copy_entire_file() or copy_file() */ +# include +# include +# include +/** Copy an entire file. +@param src source file descriptor +@param dst target to append src to +@return error code (negative) +@retval 0 on success */ +inline int copy_entire_file(int src, int dst) +{ + return fcopyfile(src, dst, NULL, COPYFILE_ALL | COPYFILE_CLONE); +} +#else +# ifdef __cplusplus +extern "C" +# endif +/** Copy an entire file. +@param src source file descriptor +@param dst target to append src to +@return error code (non-positive) +@retval 0 on success */ +int copy_entire_file(int src, int dst); +#endif + +#ifdef __cplusplus +extern "C" +#endif +/** Copy a portion of a file. +@param src source file descriptor +@param dst target to append src to +@param start first offset to copy +@param end last offset to copy (exclusive) +@return error code (non-positive) +@retval 0 on success */ +int copy_file(IF_WIN(const native_file_handle&,int) src, + IF_WIN(const native_file_handle&,int) dst, + uint64_t start, uint64_t end); + +#ifdef __cplusplus +extern "C" +#endif +/** Append to the configuration file. +@param target backup target directory +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +int backup_config_append(IF_WIN(const char*, int) target, + const char *config, size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** Append to the configuration file. +@param target backup stream +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +int backup_stream_config(IF_WIN(HANDLE, int) stream, + const char *config, size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** Start streaming a file. +@param target backup target +@param name file name +@param mode file access mode +@param size physical length of the file, in bytes +@param chunks payload chunks of a sparse file, or nullptr +@param n_chunks number of chunks; 0 unless sparse file +@return error code (non-positive) +@retval 0 on success */ +int backup_stream_start(IF_WIN(HANDLE, int) stream, + const char *name, mode_t mode, uint64_t size, + const struct backup_chunk *chunks, size_t n_chunks); + +#ifdef __cplusplus +extern "C" +#endif +/** + Write data to a stream. + @param stream backup stream + @param buf source buffer + @param size length of the buffer (usually an integer multiple of 512) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_write(IF_WIN(HANDLE, int) stream, const void *buf, + size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** + Append a file snippet to the stream, + after a corresponding call to backup_stream_start(). + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_append(IF_WIN(const native_file_handle&,int) src, + IF_WIN(HANDLE, int) stream, + uint64_t start, uint64_t end); + +#ifdef __linux__ +# ifdef __cplusplus +extern "C" +# endif +/** + Append an immutable snippet of a file to the stream, + allowing Linux sendfile(2) to be invoked. + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_append_async(int src, int stream, + uint64_t start, uint64_t end); +#elif defined _WIN32 +# define backup_stream_append_async backup_stream_append_plain +#else +# define backup_stream_append_async backup_stream_append +#endif + +#ifdef _WIN32 +# ifdef __cplusplus +extern "C" +# endif +int backup_stream_append_plain(HANDLE src, HANDLE stream, + uint64_t start, uint64_t end); +#else +# define backup_stream_append_plain backup_stream_append +#endif diff --git a/sql/sql_command.h b/sql/sql_command.h index 9c9166706a034..b8903399711f0 100644 --- a/sql/sql_command.h +++ b/sql/sql_command.h @@ -103,6 +103,7 @@ enum enum_sql_command { SQLCOM_SHOW_PACKAGE_BODY_CODE, SQLCOM_BACKUP, SQLCOM_BACKUP_LOCK, SQLCOM_SHOW_CREATE_SERVER, + SQLCOM_BACKUP_SERVER, /* When a command is added here, be sure it's also added in mysqld.cc diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index bcd6de564f550..b549017622332 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -781,6 +781,7 @@ void init_update_queries(void) sql_command_flags[SQLCOM_DROP_SERVER]|= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP]= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP_LOCK]= CF_AUTO_COMMIT_TRANS; + sql_command_flags[SQLCOM_BACKUP_SERVER]= CF_AUTO_COMMIT_TRANS; /* The following statements can deal with temporary tables, @@ -5899,6 +5900,7 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt) case SQLCOM_CALL: case SQLCOM_REVOKE: case SQLCOM_GRANT: + case SQLCOM_BACKUP_SERVER: if (thd->variables.option_bits & OPTION_IF_EXISTS) lex->create_info.set(DDL_options_st::OPT_IF_EXISTS); DBUG_ASSERT(lex->m_sql_cmd != NULL); @@ -10254,7 +10256,7 @@ int test_if_data_home_dir(const char *dir) if (!dir) DBUG_RETURN(0); - (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH); + (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH|MY_RESOLVE_SYMLINKS); DBUG_RETURN(path_starts_from_data_home_dir(path)); } diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index b2071b1c50b7f..08bf1fb7d84c8 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -50,6 +50,7 @@ #include "sql_alter.h" // Sql_cmd_alter_table* #include "sql_truncate.h" // Sql_cmd_truncate_table #include "sql_admin.h" // Sql_cmd_analyze/Check..._table +#include "sql_backup.h" #include "sql_partition_admin.h" // Sql_cmd_alter_table_*_part. #include "sql_handler.h" // Sql_cmd_handler_* #include "sql_signal.h" @@ -1473,7 +1474,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %type json_on_response %type json_type_constraint -%type json_key_unique_constraint +%type json_key_unique_constraint opt_concurrent %type json_predicate %type field_type field_type_all field_type_all_builtin @@ -15563,6 +15564,31 @@ backup_statements: /* Table list is empty for unlock */ Lex->sql_command= SQLCOM_BACKUP_LOCK; } + | SERVER_SYM TO_SYM TEXT_STRING_sys opt_concurrent + { + Lex->sql_command= SQLCOM_BACKUP_SERVER; + Lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_backup($3, $4); + } + | SERVER_SYM WITH opt_concurrent TEXT_STRING_sys + { + Lex->sql_command= SQLCOM_BACKUP_SERVER; + Lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_backup($3, $4); + } + ; + +opt_concurrent: + /* empty */ + { $$= 1; } + | ulonglong_num CONCURRENT + { + $$= int($1); + if ($1 < 1 || $1 > 256) + { + my_error(ER_DATA_OUT_OF_RANGE, myf(0), "CONCURRENT", + "BACKUP SERVER"); + MYSQL_YYABORT; + } + } ; opt_delete_gtid_domain: diff --git a/sql/sys_vars.inl b/sql/sys_vars.inl index 0f8aa1eb63bf6..8e5f0b984e81e 100644 --- a/sql/sys_vars.inl +++ b/sql/sys_vars.inl @@ -2506,6 +2506,7 @@ public: bool session_update(THD *thd, set_var *var) override; }; +#ifdef HAVE_REPLICATION /* Class for replicate_events_marked_for_skip. We need a custom update function that ensures the slave is stopped when @@ -2647,6 +2648,7 @@ public: } const uchar *global_value_ptr(THD *thd, const LEX_CSTRING *base) const override; }; +#endif /* HAVE_REPLICATION */ /** diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 9e3a23b34ab46..d63751a16af08 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -185,6 +185,8 @@ SET(INNOBASE_SOURCES handler/handler0alter.cc handler/innodb_binlog.cc handler/i_s.cc + handler/backup_innodb.h + handler/backup_innodb.cc ibuf/ibuf0ibuf.cc include/btr0btr.h include/btr0btr.inl diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 756a8b85d4752..50fb2f69d4a25 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -35,6 +35,7 @@ Created 11/11/1995 Heikki Tuuri #include "buf0buf.h" #include "buf0checksum.h" #include "buf0dblwr.h" +#include "backup_innodb.h" #include "srv0start.h" #include "page0zip.h" #include "fil0fil.h" @@ -1000,6 +1001,13 @@ uint32_t fil_space_t::flush_freed(bool writable) noexcept mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex); mysql_mutex_assert_not_owner(&buf_pool.mutex); + /* Note: There is no need to invoke writing_start() or + writing_stop() here, because we are only overwriting freed (garbage) + pages. If backup reads a torn page, it will also have copied a + corresponding FREE_PAGE record, which would be applied on recovery. + Besides, the freed page should never be reachable from other pages + that are part of the snapshot. */ + const bool punch_hole= chain.start->punch_hole == 1; if (!punch_hole && !srv_immediate_scrub_data_uncompressed) return 0; @@ -1275,6 +1283,16 @@ ATTRIBUTE_COLD static size_t buf_flush_LRU_to_withdraw(size_t to_withdraw, return to_withdraw; } +/** Stop writing to a tablespace. +@param space tablespace +@return nullptr */ +static fil_space_t *writing_stop(fil_space_t *space) noexcept +{ + space->writing_stop(); + space->release(); + return nullptr; +} + /** Flush dirty blocks from the end buf_pool.LRU, and move clean blocks to buf_pool.free. @param max maximum number of blocks to flush @@ -1292,6 +1310,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1371,7 +1390,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, buf_pool.lru_hp.set(bpage); mysql_mutex_unlock(&buf_pool.mutex); if (space) - space->release(); + writing_stop(space); auto p= buf_flush_space(space_id); space= p.first; last_space_id= space_id; @@ -1380,6 +1399,10 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, mysql_mutex_lock(&buf_pool.mutex); goto no_space; } + + backup_page_end= space->writing_start() + ? space->backup_page_end() : 0; + mysql_mutex_lock(&buf_pool.mutex); buf_pool.stat.n_pages_written+= p.second; } @@ -1391,8 +1414,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, } else if (space->is_stopping_writes()) { - space->release(); - space= nullptr; + space= writing_stop(space); no_space: mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_discard_page(bpage); @@ -1409,7 +1431,8 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, break; } - if (neighbors && space->is_rotational() && UNIV_LIKELY(!to_withdraw) && + if (neighbors && UNIV_LIKELY(!(to_withdraw | backup_page_end)) && + space->is_rotational() && /* Skip neighbourhood flush from LRU list if we haven't yet reached half of the free page target. */ UT_LIST_GET_LEN(buf_pool.free) * 2 >= free_limit) @@ -1421,10 +1444,17 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, flush: if (UNIV_UNLIKELY(to_withdraw != 0)) to_withdraw= buf_flush_LRU_to_withdraw(to_withdraw, *bpage); - if (bpage->flush(space)) + const uint32_t page{bpage->id().page_no()}; + if (page < backup_page_end && + page >= backup_page_end - space->BACKUP_BATCH_SIZE) + bpage->lock.u_unlock(true); + else if (bpage->flush(space)) + { ++n->flushed; - else - continue; + goto reacquire_mutex; + } + + continue; } goto reacquire_mutex; @@ -1437,7 +1467,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, buf_pool.lru_hp.set(nullptr); if (space) - space->release(); + writing_stop(space); if (scanned) { @@ -1484,6 +1514,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1555,10 +1586,12 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); mysql_mutex_unlock(&buf_pool.mutex); if (space) - space->release(); + writing_stop(space); auto p= buf_flush_space(space_id); space= p.first; last_space_id= space_id; + backup_page_end= space && space->writing_start() + ? space->backup_page_end() : 0; mysql_mutex_lock(&buf_pool.mutex); buf_pool.stat.n_pages_written+= p.second; mysql_mutex_lock(&buf_pool.flush_list_mutex); @@ -1567,10 +1600,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept ut_ad(!space); } else if (space->is_stopping_writes()) - { - space->release(); - space= nullptr; - } + space= writing_stop(space); if (!space) buf_flush_discard_page(bpage); @@ -1579,9 +1609,17 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); do { - if (neighbors && space->is_rotational()) + if (neighbors && UNIV_LIKELY(!backup_page_end) && + space->is_rotational()) count+= buf_flush_try_neighbors(space, page_id, bpage, neighbors == 1, count, max_n); + else if (page_id.page_no() < backup_page_end && + page_id.page_no() >= + backup_page_end - space->BACKUP_BATCH_SIZE) + { + bpage->lock.u_unlock(true); + continue; + } else if (bpage->flush(space)) ++count; else @@ -1600,7 +1638,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept buf_pool.flush_hp.set(nullptr); if (space) - space->release(); + writing_stop(space); if (scanned) { @@ -1691,6 +1729,7 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept if (written) buf_pool.stat.n_pages_written+= written; } + mysql_mutex_lock(&buf_pool.flush_list_mutex); for (buf_page_t *bpage= UT_LIST_GET_LAST(buf_pool.flush_list); bpage; ) @@ -1733,17 +1772,35 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept acquired= false; goto was_freed; } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (bpage->flush(space)) + uint32_t page, backup_page_end; + + if (UNIV_UNLIKELY(space->writing_start())) { - ++n_flush; - if (!--max_n_flush) + page= bpage->id().page_no(); + backup_page_end= space->backup_page_end(); + if (page < backup_page_end && + page >= backup_page_end - space->BACKUP_BATCH_SIZE) { + bpage->lock.u_unlock(true); + space->writing_stop(); + skip: mysql_mutex_lock(&buf_pool.mutex); mysql_mutex_lock(&buf_pool.flush_list_mutex); may_have_skipped= true; goto done; } + } + + const bool written{bpage->flush(space)}; + space->writing_stop(); + + if (written) + { + ++n_flush; + if (!--max_n_flush) + goto skip; mysql_mutex_lock(&buf_pool.mutex); } } @@ -2060,6 +2117,7 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept this->end_lsn= end_lsn; if (!archive) { + archived_checkpoint= checkpoint; archived_lsn= end_lsn; checkpoint_completed: if (resize_log.m_file == log.m_file) @@ -2077,7 +2135,7 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept /* Make the previous archived log file read-only */ #ifdef _WIN32 resize_log.close(); - SetFileAttributesA(get_archive_path().c_str(), + SetFileAttributesA(get_archive_path(get_first_lsn() - capacity()).c_str(), FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_ARCHIVE); #else struct stat st; @@ -2087,9 +2145,10 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept st.st_mode= 0444; if (fchmod(resize_log.m_file, st.st_mode)) my_error(ER_ERROR_ON_CLOSE, MYF(ME_ERROR_LOG), - get_archive_path().c_str(), errno); + get_archive_path(get_first_lsn() - capacity()).c_str(), errno); resize_log.close(); #endif + innodb_backup_checkpoint(); } else goto checkpoint_completed; diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index d8976dec66974..fe221f72f0873 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -889,6 +889,10 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) dict_sys.lock(SRW_LOCK_CALL); + if (fil_system.have_all_spaces) { + goto done; + } + if (!spaces && !upgrade && !encryption_key_id_exists(FIL_DEFAULT_ENCRYPTION_KEY)) { max_space_id = dict_find_max_space_id(&pcur, &mtr); @@ -985,6 +989,7 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) ut_free(filepath); } + fil_system.have_all_spaces = true; done: mtr.commit(); diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc new file mode 100644 index 0000000000000..f549248202802 --- /dev/null +++ b/storage/innobase/handler/backup_innodb.cc @@ -0,0 +1,1113 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "sql_class.h" +#include "backup_innodb.h" +#include "sql_backup_interface.h" +#include "trx0trx.h" +#include "buf0flu.h" +#include "log0crypt.h" +#include "dict0load.h" +#include +#ifdef __linux__ +# include +# include +#endif + +/** Associate a transaction with the current session +@param thd session +@return InnoDB transaction */ +trx_t *check_trx_exists(THD *thd) noexcept; + +namespace +{ +/** Backup state; protected by log_sys.latch */ +class InnoDB_backup +{ + /** Backup context */ + struct context + { + /** Start LSN of the first backed up log file */ + lsn_t first_lsn; + /** Start LSN of the last log file, or LSN_MAX if not determined yet */ + lsn_t max_first_lsn; + /** Final LSN of the backup, or LSN_MAX if not determined yet */ + lsn_t last_lsn; + /** size of the first log file */ + uint64_t first_size; + /** Checkpoint at the start of the backup */ + lsn_t checkpoint; + /** Log record pointing to the checkpoint */ + lsn_t checkpoint_end_lsn; + /** the original state of innodb_log_archive before/after backup */ + bool archived; + /** whether end() was invoked */ + bool cleaned_up; + /** the start LSN of the last hard-linked file, or 0 */ + std::atomic last_hardlink; + + /** + Note that a log file was hard-linked. + @param lsn start LSN of a hard-linked file + */ + void note_hardlink(lsn_t lsn) noexcept + { + for (lsn_t last= last_hardlink.load(std::memory_order_relaxed); + last < lsn && !last_hardlink. + compare_exchange_weak(last, lsn, + std::memory_order_relaxed, + std::memory_order_relaxed); ) {} + } + + /** Ensure that the last, hard-linked log file is not shared with + the server data directory, by copying it until the final LSN + @param target backup target directory + @param hl last_hardlink + @return error code + @retval 0 on success + */ + ATTRIBUTE_COLD int de_hardlink(const backup_target &target, lsn_t hl) + noexcept + { +#ifdef _WIN32 + std::string src{target.path}; + src.push_back('/'); + std::string dst{src}; + src.append("ib_logfile101"); + log_sys.append_archive_name(dst, hl); + const char *const s_{src.c_str()}, *const d_{dst.c_str()}; + if (!MoveFileEx(d_, s_, 0)) + { + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), d_, s_, errno); + return 1; + } + HANDLE s, d; + for (;;) + { + s= CreateFile(s_, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (s != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + my_osmaperr(GetLastError()); + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), s_, errno); + return 1; + } + d= CreateFile(d_, GENERIC_WRITE, 0, my_win_file_secattr(), + CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); + if (d == INVALID_HANDLE_VALUE) + { + error_return: + my_osmaperr(GetLastError()); + std::ignore= CloseHandle(s); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), s_, d_, errno); + return 1; + } +#else + std::string dst; + log_sys.append_archive_name(dst, hl); + const char *const d_{dst.c_str()}; + int d{-1}; + int err= ER_FILE_NOT_FOUND; + int s= openat(target.fd, d_, O_RDONLY); + if (s == -1) + { + error_return: + my_error(err, MYF(ME_ERROR_LOG), d_, errno); + if (s != -1) + std::ignore= close(s); + return 1; + } + err= ER_CANT_DELETE_FILE; + if (unlinkat(target.fd, d_, 0)) + goto error_return; + err= ER_CANT_CREATE_FILE; + d= openat(target.fd, d_, O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (d < 0) + goto error_return; +#endif + const uint64_t end{log_sys.START_OFFSET + last_lsn - hl}; + /* First, extend the file to a valid size. */ +#ifdef _WIN32 + int f; + { + LARGE_INTEGER li; + li.QuadPart= std::max(log_sys.FILE_SIZE_MIN, + (end + 4095) & ~4095ULL); + f= !SetFilePointerEx(d, li, nullptr, FILE_BEGIN) || !SetEndOfFile(d); + } +#else + int f= + ftruncate(d, std::max(log_sys.FILE_SIZE_MIN, + (end + 4095) & ~4095LL)); +#endif + if (!f && + !(f= copy_file(s, d, log_sys.START_OFFSET + + (hl == first_lsn) * (checkpoint - hl), end)) && + hl == first_lsn) + { + uint64_t cp_buf[8]{}; + write_checkpoint_buf(cp_buf, + checkpoint_end_lsn - hl + log_sys.START_OFFSET); + f= write_checkpoint(d, cp_buf); + } + if (IF_WIN(!CloseHandle(d), close(d)) | f) + goto error_return; + std::ignore= IF_WIN(CloseHandle(s), close(s)); +#ifdef _WIN32 + if (!DeleteFile(s_)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_DELETE_FILE, MYF(ME_ERROR_LOG), s_, errno); + return 1; + } +#endif + return 0; + } + + /** + Finish a backup. + @param target backup target + @param sink backup worker context + @return error code + @retval 0 on success + */ + int cleanup(const backup_target &target, const backup_sink &sink) noexcept + { + const lsn_t hl{last_hardlink.load(std::memory_order_relaxed)}; + if (hl == LSN_MAX) + return 0; + log_sys.latch.rd_lock(); + const lsn_t current_first_lsn{log_sys.get_first_lsn()}; + log_sys.latch.rd_unlock(); + if (hl == current_first_lsn) + { + ut_ad(sink.stream == sink.NO_STREAM); + if (int fail= de_hardlink(target, hl)) + return fail; + } + return write_config(target, sink); + } + }; + + /** pointer to backup context, or nullptr if no backup is active */ + context *ctx; + + /** the original innodb_log_file_size, or 0 */ + uint64_t old_size; + + /** collection of files to be copied */ + std::vector queue; + /** collection of completed log archive files to be + hard-linked, copied, or moved */ + std::vector logs; + +public: + /** + Start of BACKUP SERVER: collect all files to be backed up + @param thd current session + @return ctx + @retval -1 on failure + */ + void *init(THD *thd) noexcept + { + log_sys.latch.wr_lock(); + ut_ad(!ctx); + ut_ad(queue.empty()); + if (!logs.empty()) + { + /* A new BACKUP SERVER is being invoked before a previous one + had been fully finalized. Clean up any log files. */ + delete_logs(); + logs.clear(); + } + + const bool fail{log_sys.backup_start(&old_size, thd)}; + + if (!fail) + { + lsn_t start_end; + const lsn_t start= +#if 1 /* TODO: for incremental backup, allow the start to be specified */ + log_sys.get_latest_checkpoint(start_end); +#else + log_sys.archived_checkpoint; + start_end= log_sys.archived_lsn; +#endif + ctx= new context{ + log_sys.get_first_lsn(), LSN_MAX, LSN_MAX, log_sys.file_size, + start, start_end, !old_size, false, 0 + }; + + /* Collect all tablespaces that have been created before our + start checkpoint. Newer tablespaces will be recovered by the + innodb_log_archive=ON recovery. + + If a tablespace is deleted before step() is invoked, the file + will not be copied, and a FILE_DELETE record in the log will + ensure correct recovery. + + If a tablespace is renamed between this and end(), the recovery + of a FILE_RENAME record will ensure the correct file name, + no matter which name was used by step(). */ + mysql_mutex_lock(&fil_system.mutex); + for (fil_space_t &space : fil_system.space_list) + if (space.id < SRV_SPACE_ID_UPPER_BOUND && + !space.is_being_imported() && + /* FIXME: how to initialize create_lsn for old files, to + have efficient incremental backup? + fil_node_t::read_page0() cannot assign it from + FIL_PAGE_LSN because that would not reflect the file + creation but for example allocating or freeing a page. + + The easy parts of initializing space->create_lsn are + as follows: + (1) In log_parse_file() when processing FILE_CREATE + (2) In deferred_spaces.create() */ + space.get_create_lsn() < start) + queue.emplace_back(space.id); + mysql_mutex_unlock(&fil_system.mutex); + } + log_sys.latch.wr_unlock(); + DEBUG_SYNC(thd, "innodb_backup_start"); + return fail ? reinterpret_cast(-1) : ctx; + } + + /** + Process a file that was collected at init(). + This may be invoked from multiple concurrent threads. + @param target backup target + @param phase backup phase + @param sink backup worker context + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int step(const backup_target &target, backup_phase phase, + const backup_sink &sink) noexcept + { + uint32_t id{FIL_NULL}; + lsn_t lsn{0}; + log_sys.latch.wr_lock(); + const lsn_t first{log_sys.get_first_lsn()}; + ut_ad(sink.ha_data); + ut_ad(ctx ? ctx == sink.ha_data + : phase == BACKUP_PHASE_FINISH || phase == BACKUP_PHASE_NO_COMMIT); + ut_ad(static_cast(sink.ha_data)->last_lsn == LSN_MAX + ? phase == BACKUP_PHASE_START : !ctx); + size_t size{queue.size()}; + ut_ad(!size || phase == BACKUP_PHASE_START); + if (!logs.empty()) + { + lsn= logs.back(); + logs.pop_back(); + if (!size) + size= logs.size(); + } + else if (size) + { + ut_ad(phase == BACKUP_PHASE_START); + size--; + id= queue.back(); + queue.pop_back(); + } + log_sys.latch.wr_unlock(); + + if (lsn) + { + if (UNIV_UNLIKELY(lsn > first)) + /* Wait for checkpoint_complete(). */ + buf_flush_sync_batch(lsn, true); + if (replicate(lsn, target, sink, lsn < first)) + return -1; + } + else if (fil_space_t *space= fil_space_t::get(id)) + { + int res= -1; + uint32_t start{0}; +#ifdef _WIN32 + if (sink.stream == sink.NO_STREAM) + { + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain); node; + start+= node->size, node= UT_LIST_GET_NEXT(chain, node)) + if ((res= backup(target.path, node, start))) + break; + } + else + { + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain); node; + start+= node->size, node= UT_LIST_GET_NEXT(chain, node)) + if ((res= stream(sink.stream, node, start))) + break; + } +#else + int fd; + int (*method)(int, fil_node_t *, uint32_t start); + if (sink.stream == sink.NO_STREAM) + { + fd= target.fd; + method= backup; + } + else + { + fd= sink.stream; + method= stream; + } + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain); node; + start+= node->size, node= UT_LIST_GET_NEXT(chain, node)) + if ((res= (*method)(fd, node, start))) + break; +#endif + space->release(); + if (res) + return res; + } + + size= std::min(size_t{std::numeric_limits::max()}, size); + return int(size); + } + + /** + Determine the logical time of the backup snapshot. + */ + void commit() noexcept + { + log_sys.latch.wr_lock(); + ut_ad(queue.empty()); + ut_ad(ctx); + ut_ad(ctx->last_lsn == LSN_MAX); + const lsn_t last_lsn{log_sys.get_lsn()}; + lsn_t lsn{log_sys.get_first_lsn()}; + if (logs.empty() || logs.back() != lsn) + { + /* Schedule the remaining log for copying */ + logs.emplace_back(lsn); + const lsn_t next_lsn{lsn + log_sys.capacity()}; + if (next_lsn < last_lsn) + logs.emplace_back(lsn= next_lsn); + } + ctx->max_first_lsn= lsn; + ctx->last_lsn= last_lsn; + ctx= nullptr; /* unsubscribe to checkpoint_complete() */ + log_sys.latch.wr_unlock(); + } + + /** + Finish copying or finalize the backup. + @param thd current session + @param phase backup phase + @param sink backup worker context + @return error code + @retval 0 on success + */ + int end(THD *thd, backup_phase phase, const backup_sink &sink) noexcept + { + context *const ctx{static_cast(sink.ha_data)}; + if (ctx->cleaned_up) + return 0; + ctx->cleaned_up= true; + if (phase == BACKUP_PHASE_ABORT) + ctx->last_hardlink.store(LSN_MAX, std::memory_order_relaxed); + log_sys.latch.wr_lock(); + ut_ad(!this->ctx || this->ctx == ctx); + this->ctx= nullptr; /* fini() will delete the object */ + ut_ad(!log_sys.resize_in_progress()); + ut_ad(log_sys.archive); + queue.clear(); + int fail{0}; + if (!old_size) + logs.clear(); + else + { + delete_logs(); + logs.clear(); + log_sys.latch.wr_unlock(); + fail= log_sys.backup_stop_archiving(thd); + log_sys.latch.wr_lock(); + } + + log_sys.backup_stop(old_size, thd); + return fail; + } + + /** + Clean up after end(). + @param target backup target + @param sink backup worker context + @return error code + @retval 0 on success + */ + int fini(const backup_target &target, const backup_sink &sink) noexcept + { + if (context *ctx{static_cast(sink.ha_data)}) + { + ut_ad(ctx != this->ctx); + int fail{ctx->cleanup(target, sink)}; + delete ctx; + return fail; + } + return 0; + } + + /** + Complete the first checkpoint in a new archive log file. + */ + void checkpoint_complete() noexcept + { + ut_ad(log_sys.latch_have_wr()); + if (ctx) + logs.emplace_back(log_sys.get_first_lsn() - log_sys.capacity()); + } + +private: + /** Safely start backing up a tablespace file + @param end last page to copy */ + static void backup_batch_start(fil_space_t *space, uint32_t end) noexcept + { + if (space->backup_start(end)) + os_aio_wait_until_no_pending_writes(false); + } + /* Stop backing up a tablespace */ + static void backup_batch_stop(fil_space_t *space) noexcept + { space->backup_stop(); } + + /** + Delete unnecessary logs that had been created for backup. + */ + void delete_logs() noexcept + { + ut_ad(log_sys.latch_have_wr()); + ut_ad(old_size); + const lsn_t first_lsn{log_sys.get_first_lsn()}; + for (const lsn_t lsn : logs) + if (lsn != first_lsn) + IF_WIN(DeleteFile,unlink)(log_sys.get_archive_path(lsn).c_str()); + } + + /** + Back up a persistent InnoDB data file. + @param target backup target directory + @param node InnoDB data file + @param start first page number + @return error code (non-positive) + @retval 0 on success + */ + static int backup(IF_WIN(const char *,int) target, + fil_node_t *node, uint32_t start) noexcept + { + for (bool tried_mkdir{false};;) + { +#ifdef _WIN32 + std::string path{target}; + path.push_back('/'); + path.append(node->name); + HANDLE f= CreateFile(path.c_str(), GENERIC_WRITE, 0, + my_win_file_secattr(), CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (f == INVALID_HANDLE_VALUE) + { + unsigned long err= GetLastError(); + if (err == ERROR_PATH_NOT_FOUND && !tried_mkdir && + node->space->id && !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + path.erase(path.rfind('/')); + if (CreateDirectory(path.c_str(), + my_dir_security_attributes.lpSecurityDescriptor + ? &my_dir_security_attributes : nullptr) || + (err= GetLastError()) == ERROR_ALREADY_EXISTS) + continue; + } + + my_osmaperr(err); + goto fail; + } +#else + int f; +# ifdef __APPLE__ + /* aio::synchronous() in another thread may concurrently invoke + pwrite(2) on node->handle. We assume that both pwrite(2) and + fclonefileat(2) are atomic with respect to each other. Should + this assumption be invalid, some data files in the backup may be + corrupted. This corruption can be fixed by either removing this + special handling, or by implementing file-level locking. */ + f= fclonefileat(node->handle, target, node->name, 0); + if (!f) + break; + switch (errno) { + case ENOENT: + goto try_mkdir; + case ENOTSUP: + break; + default: + goto fail; + } +# endif + f= openat(target, node->name, + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (f < 0) + { + if (errno == ENOENT) + { +# ifdef __APPLE__ + try_mkdir: +# endif + if (!tried_mkdir && node->space->id && + !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + const char *sep= strchr(node->name, '/'); + ut_ad(sep); + sep= strchr(sep + 1, '/'); + ut_ad(sep); + std::string dir{node->name, size_t(sep - node->name)}; + if (!mkdirat(target, dir.c_str(), 0777) || errno == EEXIST) + continue; + } + } + goto fail; + } +#endif + int err{0}; + for (const uint32_t file_size{node->size}, + page_size{node->space->physical_size()};;) + { + const uint32_t end{start + fil_space_t::BACKUP_BATCH_SIZE}; + backup_batch_start(node->space, end); + /* TODO: avoid copying freed page ranges */ + err= copy_file(node->handle, f, start * uint64_t{page_size}, + std::min(end, file_size) * uint64_t{page_size}); + backup_batch_stop(node->space); + if (err | ((start= end) >= file_size)) + break; + } + + if (IF_WIN(!CloseHandle(f), close(f)) | err) + goto fail; + break; + } + return 0; + fail: + my_error(ER_CANT_CREATE_FILE, MYF(0), node->name, errno); + return -1; + } + + /** + Stream a persistent InnoDB data file. + @param stream backup target stream + @param node InnoDB data file + @param start first page number + @return error code (non-positive) + @retval 0 on success + */ + static int stream(IF_WIN(HANDLE,int) stream, fil_node_t *node, + uint32_t start) noexcept + { + const uint32_t file_size{node->size}, + page_size{node->space->physical_size()}; + int err= backup_stream_start(stream, node->name, 0644, + uint64_t{file_size} * page_size, + /* TODO: leave holes for freed page ranges */ + nullptr, 0); + while (!err && start < file_size) + { + const uint32_t end{start + fil_space_t::BACKUP_BATCH_SIZE}; + backup_batch_start(node->space, end); + err= backup_stream_append(node->handle, stream, + start * uint64_t{page_size}, + std::min(end, file_size) * + uint64_t{page_size}); + backup_batch_stop(node->space); + start= end; + } + + if (err) + my_error(ER_IO_WRITE_ERROR, MYF(0), errno, strerror(errno), + "BACKUP SERVER"); + return err; + } + +private: + /** + Initialize a checkpoint header buffer pointing to the start of the backup. + @param buf checkpoint buffer + @param c offset of the FILE_CHECKPOINT mini-transaction + */ + static void write_checkpoint_buf(uint64_t *buf, uint64_t c) noexcept + { + ut_ad(c >= log_sys.START_OFFSET); + if (log_sys.is_encrypted()) + log_crypt_write_header(reinterpret_cast(buf), true); + buf[4 * log_sys.is_encrypted()]= my_htobe64(c); + } + + /** Write a checkpoint header pointing to the start of the backup. + @param dst target file + @param buf checkpoint header + @return error code + @retval 0 on success */ + static int write_checkpoint(IF_WIN(HANDLE,int) dst, const void *buf) noexcept + { +#ifdef _WIN32 + using tpool::pwrite; +#endif + for (ssize_t o= 0, count= 64; count;) + { + ssize_t ret= + pwrite(dst, static_cast(buf) + o, count, o); + if (ret <= 0 || ret > count) + return -1; + o+= ret; + count-= ret; + } + return 0; + } + +public: + /** Maximum length of the configuration string */ + static constexpr size_t CONFIG_SIZE= + sizeof "[server]\n# checkpoint=" + + sizeof "innodb_log_recovery_start=" + + sizeof "innodb_log_recovery_target=\n" + 45 * 3; + + /** Write the configuration parameters for restoring the backup + @param config buffer for configuration string + @param ctx backup context + @return size of the configuration string */ + static size_t write_config_buf(char *config, const context &ctx) + noexcept + { + ut_ad(ctx.last_lsn != LSN_MAX); + return size_t(snprintf(config, CONFIG_SIZE, + "[server]\n# checkpoint=" LSN_PF "\n" + "innodb_log_recovery_start=" LSN_PF "\n" + "innodb_log_recovery_target=" LSN_PF "\n", + ctx.checkpoint, ctx.checkpoint_end_lsn, + ctx.last_lsn)); + } + + /** Write the configuration parameters for restoring the backup + @param target backup target + @param sink backup worker context + @param ctx backup context + @return error code (non-positive) + @retval 0 on success */ + static int write_config(const backup_target &target, + const backup_sink &sink) noexcept + { + char config[CONFIG_SIZE]; + const size_t size + {write_config_buf(config, *static_cast(sink.ha_data))}; + return sink.stream == sink.NO_STREAM + ? backup_config_append(IF_WIN(target.path, target.fd), config, size) + : backup_stream_config(sink.stream, config, size); + } + + /** + Hard-link (copy) or rename (move) or stream an archive log file. + @param lsn The first LSN in the file + @param target backup target + @param sink backup context + @param old lsn < log_sys.get_first_lsn() + @return error code + @retval 0 on success + */ + static int replicate(lsn_t lsn, + const backup_target &target, + const backup_sink &sink, bool old) noexcept + { + ut_ad(log_get_lsn() >= lsn); + const std::string p{log_sys.get_archive_path(lsn)}; + const char *const path= p.c_str(), *basename= strrchr(path, '/'); + if (!basename) + basename= path; + else + basename++; + context &ctx{*static_cast(sink.ha_data)}; + const bool move{old && !ctx.archived}; + uint64_t cp_buf[8]{}; +#ifdef _WIN32 + ut_ad(!target.path == (sink.stream != sink.NO_STREAM)); + std::string b; + const char *destname= nullptr; + if (!target.path) + goto send_file; + b= target.path; + b.push_back('/'); + b.append(basename); + destname= b.c_str(); + unsigned long err; + if (move) + { + if (!MoveFileEx(path, destname, MOVEFILE_COPY_ALLOWED)) + { + fail: + err= GetLastError(); + got_err: + my_osmaperr(err); + if (target.path) + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), path, basename, + errno); + else + my_error(ER_IO_WRITE_ERROR, MYF(ME_ERROR_LOG), + errno, strerror(errno), "BACKUP SERVER"); + return -1; + } + + if (lsn < ctx.checkpoint) + { + if (!SetFileAttributes(destname, FILE_ATTRIBUTE_NORMAL)) + goto fail; + HANDLE dh= CreateFile(destname, GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (dh == INVALID_HANDLE_VALUE) + goto fail; + if (os_file_set_sparse_win32(dh)) + std::ignore= + os_file_punch_hole(dh, 0, log_sys.START_OFFSET + + ((ctx.checkpoint - lsn) & ~4095ULL)); + write_checkpoint_buf(cp_buf, ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + int fail= write_checkpoint(dh, cp_buf); + std::ignore= CloseHandle(dh); + if (fail) + goto fail; + } + return 0; + } + else if (CreateHardLink(destname, path, nullptr)) + { + ctx.note_hardlink(lsn); + return 0; + } + + if ((err= GetLastError()) != ERROR_NOT_SAME_DEVICE) + goto got_err; + /* Hard-linking failed. Try copying with the final name. */ + if (target.path) + { + b= target.path; + b.push_back('/'); + b.append(basename); + destname= b.c_str(); + + if (lsn >= ctx.checkpoint && lsn < ctx.max_first_lsn) + { + /* Copy a middle log file entirely. */ + if (CopyFileEx(path, basename, nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING)) + return 0; + goto fail; + } + } + + send_file: + HANDLE src; + for (;;) + { + src= CreateFile(path, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (src != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + goto fail; + } + HANDLE dst{sink.stream}; + if (dst == INVALID_HANDLE_VALUE) + { + dst= CreateFile(destname, GENERIC_WRITE, 0, my_win_file_secattr(), + CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); + if (dst == INVALID_HANDLE_VALUE) + { + std::ignore= CloseHandle(src); + goto fail; + } + } +#else + if (sink.stream != sink.NO_STREAM); + else if (move + ? !renameat(AT_FDCWD, path, target.fd, basename) + : !linkat(AT_FDCWD, path, target.fd, basename, AT_SYMLINK_FOLLOW)) + { + if (!move) + ctx.note_hardlink(lsn); +# ifdef __linux__ + else if (lsn != ctx.first_lsn); + else if (off_t garbage= (ctx.checkpoint - lsn) & ~4095ULL) + /* Best effort to punch a hole to free up some garbage in + the first file. We do not care about failures. */ + if (!fchmodat(target.fd, basename, 0644, 0)) + { + int dst= openat(target.fd, basename, O_RDWR); + if (dst >= 0) + fallocate(dst, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + log_sys.START_OFFSET, garbage); + close(dst); + std::ignore= fchmodat(target.fd, basename, 0444, 0); + } +# endif + return 0; + } + else if (errno != EXDEV) + { + fail: + if (sink.stream == sink.NO_STREAM) + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), path, basename, errno); + else + my_error(ER_IO_WRITE_ERROR, MYF(ME_ERROR_LOG), errno, strerror(errno), + "BACKUP SERVER"); + return -1; + } + + const int src{open(path, O_RDONLY)}; + if (src < 0) + goto fail; + if (move && unlink(path)) + { + close_and_fail: + std::ignore= close(src); + goto fail; + } + int dst{sink.stream}; + if (dst < 0) + { + dst= openat(target.fd, basename, + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (dst < 0) + goto close_and_fail; + } + int err; +#endif + backup_chunk chunks[3], *chunk{chunks}; + *chunk++= {log_sys.START_OFFSET, ctx.last_lsn - lsn}; + if (lsn < ctx.checkpoint) + { + /* Copy the necessary part of the first log file. */ + ut_ad(lsn == ctx.first_lsn); + write_checkpoint_buf(cp_buf, ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + chunk[-1]= {0, 512}; + const lsn_t end= + std::min(ctx.last_lsn, lsn + ctx.first_size - log_sys.START_OFFSET); + *chunk++= + {log_sys.START_OFFSET + ctx.checkpoint - lsn, end - ctx.checkpoint}; + chunk->offset= end - lsn; + goto pad_size; + } + else if (lsn < ctx.max_first_lsn) + { + /* Copy a middle log file entirely. */ +#ifdef _WIN32 + ut_ad(dst == sink.stream); + chunk->offset= os_file_get_size(src); +#else + if (dst != sink.stream) + { + err= copy_entire_file(src, dst); + goto close_dst; + } + chunk->offset= uint64_t(lseek(src, 0, SEEK_END)); +#endif + /* Omit the checkpoint header from the stream. */ + chunk[-1].length= chunk->offset - log_sys.START_OFFSET; + goto stream_file; + } + else + { + ut_ad(ctx.max_first_lsn == lsn); + ut_ad(ctx.last_lsn > lsn); + ut_ad(ctx.last_lsn != LSN_MAX); + ut_ad(chunk[-1].length == ctx.last_lsn - lsn); + chunk->offset= chunk[-1].length; + pad_size: + /* Set the logical size of the file. */ + chunk->offset= + std::max(log_sys.FILE_SIZE_MIN, + (chunk->offset + (log_sys.START_OFFSET + 4095)) & + ~4095ULL); + } + + if (dst == sink.stream) + { + stream_file: + chunk++->length= 0; + const backup_chunk &end{chunk[-2]}; + ut_ad(chunk - chunks == 2 || chunk - chunks == 3); + const size_t cp_size{(size_t(chunk - chunks) & 1) << 9}; + err= backup_stream_start(dst, basename, + 0444 | int{lsn == ctx.max_first_lsn} << 7, + end.length + cp_size, chunks, chunk - chunks); + if (!err && cp_size) + err= backup_stream_write(dst, cp_buf, sizeof cp_buf) || + backup_stream_write(dst, field_ref_zero, cp_size - sizeof cp_buf); + if (!err) + { + err= backup_stream_append_async(src, dst, end.offset, + end.offset + end.length); + if (err); + else if (size_t pad= size_t(end.length) & 511) + err= backup_stream_write(dst, field_ref_zero, 512 - pad); + } + } + else + { + /* First, extend the file to a valid size. */ +#ifdef _WIN32 + LARGE_INTEGER li; + li.QuadPart= chunk->offset; + err= !SetFilePointerEx(dst, li, nullptr, FILE_BEGIN) || + !SetEndOfFile(dst) || +#else + err= ftruncate(dst, chunk->offset) || +#endif + copy_file(src, dst, chunk[-1].offset, chunk[-1].offset + + chunk[-1].length) || + (lsn < ctx.checkpoint && write_checkpoint(dst, cp_buf)); +#ifdef _WIN32 + err|= !CloseHandle(dst); +#else + close_dst: + err|= close(dst); +#endif + } + + if (err | IF_WIN(!CloseHandle(src), close(src))) + goto fail; + + return 0; + } +}; + +/** The backup context; protected by log_sys.latch */ +static InnoDB_backup innodb_backup; +} + +bool log_t::backup_start(uint64_t *old_size, THD *thd) noexcept +{ + ut_ad(latch_have_wr()); + ut_ad(!backup); + backup= true; + *old_size= 0; + if (archive) + return false; + const uint64_t old_file_size{file_size}; + latch.wr_unlock(); + const bool fail{set_archive(true, thd, true)}; + latch.wr_lock(); + if (!fail) + { + *old_size= old_file_size; + return false; + } + ut_ad(backup); + backup= false; + const uint64_t new_file_size{file_size}; + latch.wr_unlock(); + if (old_file_size != new_file_size && old_file_size && + resize_start(old_file_size, thd) == RESIZE_STARTED) + resize_finish(thd); + latch.wr_lock(); + return true; +} + +void log_t::backup_stop(uint64_t old_size, THD *thd) noexcept +{ + ut_ad(latch_have_wr()); + /* We will be invoked with old_size=0 after a failed backup_start(), + or if innodb_log_archive=ON held during a successful backup_start(). */ + ut_ad(!old_size || !resize_in_progress()); + ut_ad(!old_size || backup); + backup= false; + const uint64_t new_size{file_size}; + latch.wr_unlock(); + if (old_size && old_size != new_size && + resize_start(old_size, thd) == RESIZE_STARTED) + resize_finish(thd); +} + +void *innodb_backup_start(THD *thd, const backup_target *, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + case BACKUP_PHASE_PREPARE_START: + if (!fil_system.have_all_spaces) + { + /* To speed up startup, InnoDB does not normally open all + tablespace files that are pointed to by SYS_TABLES. + InnoDB_backup::init() assumes that the information of all + tablespaces is available, including files that had been created + before the server was started, and never opened in the course of + the current server execution. */ + dict_load_tablespaces(nullptr, true); + ut_ad(fil_system.have_all_spaces); + } + return 0; + case BACKUP_PHASE_START: + return innodb_backup.init(thd); + case BACKUP_PHASE_NO_COMMIT: + innodb_backup.commit(); + /* fall through */ + default: + return sink->ha_data; + } +} + +int innodb_backup_step(THD *, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + case BACKUP_PHASE_START: + case BACKUP_PHASE_NO_COMMIT: + case BACKUP_PHASE_FINISH: + return innodb_backup.step(*target, phase, *sink); + default: + return 0; + } +} + +int innodb_backup_end(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + default: + return 0; + case BACKUP_PHASE_FINISH: + return innodb_backup.fini(*target, *sink); + case BACKUP_PHASE_NO_COMMIT: + case BACKUP_PHASE_ABORT: + return innodb_backup.end(thd, phase, *sink); + } +} + +void innodb_backup_checkpoint() noexcept +{ + innodb_backup.checkpoint_complete(); +} diff --git a/storage/innobase/handler/backup_innodb.h b/storage/innobase/handler/backup_innodb.h new file mode 100644 index 0000000000000..e9393038e5a21 --- /dev/null +++ b/storage/innobase/handler/backup_innodb.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/** + Start of a BACKUP SERVER phase, + when no innodb_backup_step() or innodb_backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to sink, or nullptr + @retval -1 on failure +*/ +void *innodb_backup_start(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) + noexcept; + +/** + Process a file that was collected in innodb_backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @return number of files remaining, or negative on error + @retval 0 on completion +*/ +int innodb_backup_step(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept; + +/** + Finish a phase, once all calls for the current phase are completed. + @param thd current sesssion + @param target backup target + @param phase last phase on which backup_start() was successfully invoked, + or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success +*/ +int innodb_backup_end(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept; + +/** + Complete the first checkpoint in a new archive log file. +*/ +void innodb_backup_checkpoint() noexcept; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index fdf9a90f4adde..74f8f5e8e1175 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -152,6 +152,7 @@ MDL_ticket *get_mdl_ticket(TABLE *table) noexcept; #include "ha_innodb.h" #include "i_s.h" +#include "backup_innodb.h" #include #include @@ -4195,6 +4196,9 @@ static int innodb_init(void* p) = innodb_prepare_commit_versioned; innobase_hton->update_optimizer_costs= innobase_update_optimizer_costs; + innobase_hton->backup_start = innodb_backup_start; + innobase_hton->backup_step = innodb_backup_step; + innobase_hton->backup_end = innodb_backup_end; innobase_hton->binlog_init= innodb_binlog_init; innobase_hton->set_binlog_max_size= ibb_set_max_size; innobase_hton->binlog_write_direct_ordered= @@ -18871,39 +18875,7 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, ib_senderrf(thd, IB_LOG_LEVEL_ERROR, ER_CANT_CREATE_HANDLER_FILE); break; case log_t::RESIZE_STARTED: - for (timespec abstime;;) - { - if (thd_kill_level(thd)) - { - log_sys.resize_abort(thd); - break; - } - - set_timespec(abstime, 5); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - lsn_t resizing= log_sys.resize_in_progress(); - if (resizing > buf_pool.get_oldest_modification(0)) - { - buf_pool.page_cleaner_wakeup(true); - my_cond_timedwait(&buf_pool.done_flush_list, - &buf_pool.flush_list_mutex.m_mutex, &abstime); - resizing= log_sys.resize_in_progress(); - } - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (!resizing || !log_sys.resize_running(thd)) - break; - log_sys.latch.wr_lock(); - while (resizing > log_sys.get_lsn()) - { - ut_ad(!log_sys.is_mmap()); - /* The server is almost idle. Write dummy FILE_CHECKPOINT records - to ensure that the log resizing will complete. */ - mtr_t mtr{nullptr}; - mtr.start(); - mtr.commit_files(log_sys.last_checkpoint_lsn); - } - log_sys.latch.wr_unlock(); - } + log_sys.resize_finish(thd); } } mysql_mutex_lock(&LOCK_global_system_variables); @@ -19760,7 +19732,9 @@ static void innodb_log_archive_update(THD *thd, st_mysql_sys_var*, { /* MDEV-36828 TODO: On failure, report which other setting conflicted with the request */ + mysql_mutex_unlock(&LOCK_global_system_variables); log_sys.set_archive(*static_cast(save), thd); + mysql_mutex_lock(&LOCK_global_system_variables); } static MYSQL_SYSVAR_BOOL(log_archive, log_sys.archive, @@ -19773,10 +19747,20 @@ static MYSQL_SYSVAR_UINT64_T(log_archive_start, innodb_log_archive_start, "initial value of innodb_lsn_archived; 0=auto-detect", nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); +static void innodb_log_recovery_start_update(THD *, st_mysql_sys_var*, + void *, const void *save) noexcept +{ + const lsn_t lsn{*static_cast(save)}; + recv_sys.recovery_start= lsn; + if (lsn && log_sys.archive) + log_sys.archived_checkpoint= lsn; +} + static MYSQL_SYSVAR_UINT64_T(log_recovery_start, recv_sys.recovery_start, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + PLUGIN_VAR_RQCMDARG, "LSN to start recovery from (0=automatic)", - nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); + nullptr, innodb_log_recovery_start_update, + 0, 0, std::numeric_limits::max(), 0); static MYSQL_SYSVAR_UINT64_T(log_recovery_target, recv_sys.rpo, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 67fa1bfd026f0..4111d4d399d8d 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -408,6 +408,13 @@ struct fil_space_t final /** Whether any corruption of this tablespace has been reported */ mutable std::atomic_flag is_corrupted= ATOMIC_FLAG_INIT; + /** BACKUP SERVER flag in write_or_backup */ + static constexpr uint8_t BACKUP{128}; + /** whether there is a pending write or backup */ + std::atomic write_or_backup{0}; + /** first page number that is not being backed up */ + std::atomic backup_end{0}; + public: /** mutex to protect freed_ranges and last_freed_lsn */ std::mutex freed_range_mutex; @@ -1058,6 +1065,46 @@ struct fil_space_t final VALIDATE_IMPORT }; + /** Note that writes are being submitted to the tablespace. + @return whether a backup is pending */ + bool writing_start() noexcept + { + uint8_t wb{write_or_backup.fetch_add(1, std::memory_order_acq_rel)}; + ut_ad(~wb & (BACKUP - 1)); + return wb & BACKUP; + } + + /** Note that we there are no more pending writes to the tablespace. */ + void writing_stop() noexcept + { + ut_d(uint8_t wb=) write_or_backup.fetch_sub(1, std::memory_order_release); + ut_ad(wb & ~BACKUP); + } + + /** Note that we backing up some pages of the underlying files. + @param last_page the last page that is being backed up */ + bool backup_start(uint32_t last_page) noexcept + { + backup_end.store(last_page, std::memory_order_relaxed); + uint8_t wb{write_or_backup.fetch_add(BACKUP, std::memory_order_acq_rel)}; + ut_ad(!(wb & BACKUP)); + return wb & ~BACKUP; + } + /** Note that we are not currently backing up the underlying files. */ + void backup_stop() noexcept + { + backup_end.store(0, std::memory_order_relaxed); + ut_d(uint8_t wb=) + write_or_backup.fetch_sub(BACKUP, std::memory_order_release); + ut_ad(wb & BACKUP); + } + /** @return the first page number that is not being backed up */ + uint32_t backup_page_end() const noexcept + { return backup_end.load(std::memory_order_relaxed); } + + /** The size of a backup copy_file() batch in pages */ + static constexpr uint32_t BACKUP_BATCH_SIZE{64}; + /** Update the data structures on write completion */ void complete_write() noexcept; @@ -1463,6 +1510,8 @@ struct fil_system_t my_bool buffered; /** whether fdatasync() is needed on data files */ Atomic_relaxed need_unflushed_spaces; + /** whether dict_load_tablespaces(nullptr, true) is unnecessary */ + Atomic_relaxed have_all_spaces; /** Try to enable or disable write-through of data files */ void set_write_through(bool write_through); diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 44a827dbf636d..6d07fa25e013d 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -221,6 +221,8 @@ struct log_t /** whether !archive log records may have been written with get_sequence_bit()==0 */ bool circular_recovery_from_sequence_bit_0:1; + /** whether we are between backup_start() and backup_stop() */ + bool backup:1; public: /** the default value of log_mmap */ static constexpr bool log_mmap_default= @@ -288,6 +290,8 @@ struct log_t Atomic_relaxed last_checkpoint_lsn; /** The log writer (protected by latch.wr_lock()) */ lsn_t (*writer)() noexcept; + /** the earliest available checkpoint; protected by latch.wr_lock() */ + lsn_t archived_checkpoint; /** end_lsn of the first available checkpoint, or 0; protected by latch.wr_lock() */ lsn_t archived_lsn; @@ -369,11 +373,24 @@ struct log_t RESIZE_NO_CHANGE, RESIZE_IN_PROGRESS, RESIZE_STARTED, RESIZE_FAILED }; +private: /** Start resizing the log and release the exclusive latch. + @param size requested new file_size + @param thd the current thread identifier + @param backup whether the caller is backup_start() or backup_stop() + @return whether the resizing was started successfully */ + resize_start_status resize_start(uint64_t size, void *thd, bool backup) + noexcept; +public: + /** Start resizing the log. @param size requested new file_size @param thd the current thread identifier @return whether the resizing was started successfully */ - resize_start_status resize_start(os_offset_t size, void *thd) noexcept; + resize_start_status resize_start(uint64_t size, void *thd) noexcept + { return resize_start(size, thd, false); } + + /** Wait for the completion of resize_start() == RESIZE_STARTED */ + void resize_finish(THD *thd) noexcept; /** Abort a resize_start() that we started. @param thd thread identifier that had been passed to resize_start() */ @@ -397,10 +414,37 @@ struct log_t resize_write_low(lsn, end, len, seq); } +private: + /** SET GLOBAL innodb_log_archive, or start/stop BACKUP SERVER + @param archive the new value of innodb_log_archive + @param thd SQL connection + @param backup whether the caller is backup_start() or backup_stop() + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd, bool backup) noexcept; +public: /** SET GLOBAL innodb_log_archive @param archive the new value of innodb_log_archive - @param thd SQL connection */ - void set_archive(my_bool archive, THD *thd) noexcept; + @param thd SQL connection + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd) noexcept + { return set_archive(archive, thd, false); } + + /** Start BACKUP SERVER. + @param old_size the old file_size, or 0 on failure or when + already running innodb_log_archive=ON + @param thd SQL connection + @return whether the operation failed */ + bool backup_start(uint64_t *old_size, THD *thd) noexcept; + /** Stop log archiving in BACKUP SERVER clean-up + @param thd SQL connection + @return whether the operation failed */ + bool backup_stop_archiving(THD *thd) noexcept + { return set_archive(false, thd, true); } + + /** Stop BACKUP SERVER. + @param old_size the value returned by backup_start() + @param thd SQL connection */ + void backup_stop(uint64_t old_size, THD *thd) noexcept; private: /** Replicate a write to the log. @@ -695,6 +739,18 @@ struct log_t /** @return the first LSN of the log file */ lsn_t get_first_lsn() const noexcept { return first_lsn; } + /** + Determine the latest checkpoint. + @param end LSN leading to the FILE_CHECKPOINT record + @return the latest checkpoint LSN + */ + lsn_t get_latest_checkpoint(lsn_t &end) const noexcept + { + ut_ad(latch_have_any()); + end= end_lsn; + return last_checkpoint_lsn; + } + /** Set the recovered checkpoint. @param lsn log sequence number of the checkpoint @param end_lsn LSN passed to write_checkpoint() diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index ee8d43a68e500..aa2063719d8e6 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -635,7 +635,7 @@ void log_t::set_buffered(bool buffered) noexcept } #endif - /** Try to enable or disable durable writes (update log_write_through) */ +/** Try to enable or disable durable writes (update log_write_through) */ void log_t::set_write_through(bool write_through) { if (is_mmap() || high_level_read_only || recv_sys.rpo) @@ -764,9 +764,12 @@ void log_t::header_rewrite(my_bool archive) noexcept /** SET GLOBAL innodb_log_archive @param archive the new value of innodb_log_archive -@param thd SQL connection */ -void log_t::set_archive(my_bool archive, THD *thd) noexcept +@param thd SQL connection +@param backup whether the caller is backup_start() or backup_stop() +@return whether the operation failed */ +bool log_t::set_archive(my_bool archive, THD *thd, bool backup) noexcept { + bool fail= false; thd_wait_begin(thd, THD_WAIT_DISKIO); tpool::tpool_wait_begin(); lsn_t wait_lsn; @@ -780,12 +783,20 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept my_printf_error(ER_WRONG_USAGE, "SET GLOBAL innodb_log_file_size is in progress", MYF(0)); + fail: + fail= true; + wait_lsn= 0; break; } if (archive == this->archive) break; - if (thd_kill_level(thd)) - break; + if ((!backup || archive) && thd_kill_level(thd)) + goto fail; + if (!backup && this->backup) + { + my_printf_error(ER_WRONG_USAGE, "BACKUP SERVER is in progress", MYF(0)); + goto fail; + } if (resize_log.is_opened()) { @@ -894,7 +905,7 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept if (!log.is_opened()) { my_error(ER_ERROR_ON_READ, MYF(0), old_name, errno); - break; + goto fail; } } #endif @@ -919,7 +930,7 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept { my_error(ER_ERROR_ON_RENAME, MYF(0), old_name, new_name, my_errno); first_lsn= old_first_lsn; - break; + goto fail; } if (archive) @@ -951,14 +962,16 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept thd_wait_end(thd); if (wait_lsn) mtr_flush_ahead(wait_lsn); + return fail; } -/** Start resizing the log and release the exclusive latch. -@param size requested new file_size -@param thd the current thread identifier +/** Start resizing the log. +@param size requested new file_size +@param thd the current thread identifier +@param backup whether the caller is backup_start() or backup_stop() @return whether the resizing was started successfully */ -log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) - noexcept +log_t::resize_start_status log_t::resize_start(uint64_t size, void *thd, + bool backup) noexcept { ut_ad(size >= 4U << 20); ut_ad(!(size & 4095)); @@ -989,6 +1002,9 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) resize_target= size; } } + else if (!backup && this->backup) + /* backup_start() or backup_stop() is running */ + status= RESIZE_FAILED; else { lsn_t start_lsn; @@ -1090,6 +1106,44 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) return status; } +/** Wait for the completion of resize_start() == RESIZE_STARTED */ +void log_t::resize_finish(THD *thd) noexcept +{ + for (timespec abstime;;) + { + if (thd_kill_level(thd)) + { + resize_abort(thd); + break; + } + + set_timespec(abstime, 5); + mysql_mutex_lock(&buf_pool.flush_list_mutex); + lsn_t resizing= resize_in_progress(); + if (resizing > buf_pool.get_oldest_modification(0)) + { + buf_pool.page_cleaner_wakeup(true); + my_cond_timedwait(&buf_pool.done_flush_list, + &buf_pool.flush_list_mutex.m_mutex, &abstime); + resizing= resize_in_progress(); + } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + if (!resizing || !resize_running(thd)) + break; + latch.wr_lock(); + while (resizing > get_lsn()) + { + ut_ad(!is_mmap()); + /* The server is almost idle. Write dummy FILE_CHECKPOINT records + to ensure that the log resizing will complete. */ + mtr_t mtr{nullptr}; + mtr.start(); + mtr.commit_files(last_checkpoint_lsn); + } + latch.wr_unlock(); + } +} + /** Abort a resize_start() that we started. */ void log_t::resize_abort(void *thd) noexcept { diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 72b62c36d6a15..d1d5c7237599b 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2099,6 +2099,7 @@ dberr_t recv_sys_t::find_checkpoint() memset_aligned<4096>(const_cast(field_ref_zero), 0, 4096); /* Mark the redo log for upgrading. */ lsn= file_checkpoint= log_sys.last_checkpoint_lsn; + log_sys.archived_checkpoint= lsn; log_sys.set_recovered_lsn(lsn); if (rpo && rpo != lsn) { @@ -2177,7 +2178,8 @@ dberr_t recv_sys_t::find_checkpoint() log_sys.set_recovered_checkpoint(checkpoint_lsn, lsn= end_lsn, field == log_t::CHECKPOINT_1); } - if (!log_sys.last_checkpoint_lsn) + log_sys.archived_checkpoint= log_sys.last_checkpoint_lsn; + if (!log_sys.archived_checkpoint) goto got_no_checkpoint; else if (!log_sys.archived_lsn) log_sys.archived_lsn= lsn; diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 6494c5e21b96e..4ef86095f78a3 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2017,7 +2017,15 @@ os_file_create_func( ); DWORD create_flag = OPEN_EXISTING; - DWORD share_mode = read_only + /* BACKUP SERVER may invoke CreateHardLink() on a log file that + may concurrently be written to. This is why we must allow + FILE_SHARE_WRITE. This has the side effect that multiple InnoDB + instances may be concurrently started on the same log file. + However, InnoDB will not write any log before it has successfully + opened data files. As long as the multiple instances are also + opening the same InnoDB data files (such as the system tablespace), + they should fail to start up concurrently. */ + DWORD share_mode = read_only || type == OS_LOG_FILE ? FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE : FILE_SHARE_READ | FILE_SHARE_DELETE; diff --git a/storage/maria/CMakeLists.txt b/storage/maria/CMakeLists.txt index 9d528f980dc5b..974145ce8ea6c 100644 --- a/storage/maria/CMakeLists.txt +++ b/storage/maria/CMakeLists.txt @@ -45,6 +45,7 @@ SET(ARIA_SOURCES ma_init.c ma_open.c ma_extra.c ma_info.c ma_rkey.c ha_maria.h maria_def.h ma_recovery_util.c ma_servicethread.c ma_norec.c ma_crypt.c ma_backup.c + ma_backup_server.cc ma_backup_server.h ) IF(APPLE) diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 8f5e47daea728..802ad0f9fcc19 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -23,6 +23,7 @@ #include #include #include "ha_maria.h" +#include "ma_backup_server.h" #include "trnman_public.h" #include "trnman.h" @@ -3942,6 +3943,9 @@ static int ha_maria_init(void *p) maria_hton->prepare_for_backup= maria_prepare_for_backup; maria_hton->end_backup= maria_end_backup; maria_hton->update_optimizer_costs= aria_update_optimizer_costs; + maria_hton->backup_start= aria_backup_start; + //maria_hton->backup_step= aria_backup_step; + maria_hton->backup_end= aria_backup_end; /* TODO: decide if we support Maria being used for log tables */ maria_hton->flags= (HTON_CAN_RECREATE | HTON_SUPPORT_LOG_TABLES | diff --git a/storage/maria/ma_backup_server.cc b/storage/maria/ma_backup_server.cc new file mode 100644 index 0000000000000..43c473a50a7ca --- /dev/null +++ b/storage/maria/ma_backup_server.cc @@ -0,0 +1,423 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "maria_def.h" +#include "ma_backup_server.h" +#include "mysqld_error.h" +#if 1 // tc_purge(), tdc_purge() +# include "sql_class.h" +# include "table_cache.h" +#endif +#include +#include +#include +#include +#include "span.h" + +/* + Implementation of functions declatred in ma_backup.h: + BACKUP SERVER support for Aria engine +*/ + +namespace +{ + /** Backup state; protected by log_sys.latch */ + class Aria_backup + { + public: + Aria_backup()= default; + ~Aria_backup() + { +#ifndef _WIN32 + if (datadir_fd >= 0) + std::ignore= close(datadir_fd); +#endif + if (translog_purge_disabled) + translog_enable_purge(); + } + + bool initialize() noexcept + { +#ifndef _WIN32 + datadir_fd= open(maria_data_root, O_DIRECTORY); + if (datadir_fd < 0) + { + my_error(ER_CANT_READ_DIR, MYF(0), maria_data_root, errno); + return true; + } +#endif // _WIN32 + assert(!translog_purge_disabled); + translog_purge_disabled= true; + translog_disable_purge(); + return false; + } + + int end(const backup_target &target, const backup_sink &sink) noexcept + { + int ret_val= perform_backup(target, sink); + assert(translog_purge_disabled); + translog_purge_disabled= false; + translog_enable_purge(); + return ret_val; + } + private: +#ifndef _WIN32 + /** The server data directory */ + int datadir_fd{-1}; +#endif + /** whether the Aria translog_disable_purge() is in effect */ + bool translog_purge_disabled{false}; + static constexpr const char zerobuf[511]{}; + using dir_contents = std::vector; + using database_dir = std::pair; + std::vector database_dirs; + std::vector log_files; + bool have_control_file = false; + + int perform_backup(const backup_target &target, const backup_sink &sink) + noexcept + { + return scan_datadir() || copy_databases(target, sink) || + copy_control_file(target, sink) || + translog_flush(translog_get_horizon()) || + copy_logs(target, sink); + } + + ATTRIBUTE_COLD ATTRIBUTE_NOINLINE + static int dir_error(const char *name) noexcept + { + my_error(ER_CANT_READ_DIR, MYF(0), name, my_errno); + return 1; + } + + int scan_datadir() noexcept + { + const char *base_dir= maria_data_root; + MY_DIR *dir_info= my_dir(base_dir, MYF(MY_WANT_STAT)); + if (!dir_info) + return dir_error(base_dir); + int fail= 0; + for (const fileinfo &fi : + st_::span{dir_info->dir_entry, + dir_info->number_of_files}) + if ((fi.mystat->st_mode & S_IFMT) == S_IFDIR) + { + if ((fail= scan_database_dir(fi.name)) != 0) + goto func_exit; + } + else if (!strncmp(fi.name, C_STRING_WITH_LEN("aria_log."))) + log_files.emplace_back(fi.name); + else if (!strcmp(fi.name, "aria_log_control")) + have_control_file = true; + func_exit: + my_dirend(dir_info); + return fail; + } + + int scan_database_dir(const char* dir_name) noexcept + { + const char* base_dir = maria_data_root; + std::string dir_path = std::string(base_dir) + "/" + dir_name; + MY_DIR *dir_info= my_dir(dir_path.c_str(), MYF(MY_WANT_STAT)); + if (!dir_info) + return dir_error(dir_path.c_str()); + std::vector files_to_backup; + for (const fileinfo &fi : + st_::span{dir_info->dir_entry, + dir_info->number_of_files}) + if (is_db_file(fi.name)) + files_to_backup.emplace_back(fi.name); + if (!files_to_backup.empty()) + database_dirs.emplace_back(dir_name, std::move(files_to_backup)); + my_dirend(dir_info); + return 0; + } + + int copy_databases(const backup_target &target, const backup_sink &sink) + noexcept + { + for (const database_dir &dir : database_dirs) + { + if (sink.stream != sink.NO_STREAM); + else if (int fail= ensure_target_subdir(target, dir.first.c_str())) + return fail; + if (int fail= copy_database(target, sink, dir)) + return fail; + } + return 0; + } + + /* + Create directory in the target directory if it does not exist. + Return 0 on success, non-0 on failure. Set errno in case of failure + */ + int ensure_target_subdir(const backup_target &target, const char *name) + noexcept + { +#ifdef _WIN32 + if (CreateDirectory(make_path(target, name).c_str(), nullptr)) + return 0; + DWORD err= GetLastError(); + if (err == ERROR_ALREADY_EXISTS) + return 0; + my_osmaperr(err); +#else + if (likely(!mkdirat(target.fd, name, 0777) || errno == EEXIST)) + return 0; +#endif + my_error(ER_CANT_CREATE_FILE, MYF(0), name, errno); + return 1; + } + + int copy_database(const backup_target &target, const backup_sink &sink, + const database_dir& dir) noexcept + { + std::string file_path; + for (const std::string &file : dir.second) + { + file_path= dir.first; + file_path.push_back('/'); + file_path.append(file); + if (int fail= copy_file(target, sink, file_path.c_str())) + return fail; + } + return 0; + } + + int copy_control_file(const backup_target &target, const backup_sink &sink) + noexcept + { + if (!have_control_file) + return 0; + return copy_file(target, sink, "aria_log_control"); + } + + int copy_logs(const backup_target &target, const backup_sink &sink) + noexcept + { + for (const std::string &file : log_files) + if (int fail= copy_file(target, sink, file.c_str())) + return fail; + return 0; + } + + int copy_file(const backup_target &target, const backup_sink &sink, + const char *path) const noexcept + { +#ifndef _WIN32 + int ret_val{0}; + int src_fd{openat(datadir_fd, path, O_RDONLY)}; + if (src_fd < 0) + { + my_error(ER_CANT_OPEN_FILE, MYF(0), path, errno); + return 1; + } + int tgt_fd{sink.stream}; + if (tgt_fd == sink.NO_STREAM) + { + tgt_fd= openat(target.fd, path, + O_CREAT | O_EXCL | O_WRONLY, 0666); + if (tgt_fd < 0) + { + my_error(ER_CANT_CREATE_FILE, MYF(0), path, errno); + ret_val= 1; + } + else + { + ret_val= copy_entire_file(src_fd, tgt_fd); + if (ret_val | close(tgt_fd)) + { + write_error: + my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); + ret_val= 1; + } + } + } + else + { + uint64_t end= uint64_t(lseek(src_fd, 0, SEEK_END)); + if (backup_stream_start(tgt_fd, path, 0644, end, nullptr, 0) || + backup_stream_append(src_fd, tgt_fd, 0, end)) + goto write_error; + if (size_t pad= size_t(end) & 511) + if (backup_stream_write(tgt_fd, zerobuf, 512 - pad)) + goto write_error; + } + + close(src_fd); + return ret_val; +#else + std::string src_path= std::string(maria_data_root) + "/" + path; + + if (sink.stream == sink.NO_STREAM) + { + std::string dest_path{make_path(target, path)}; + if (!CopyFileEx(src_path.c_str(), dest_path.c_str(), + nullptr, nullptr, nullptr, COPY_FILE_NO_BUFFERING)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_CREATE_FILE, MYF(0), dest_path.c_str(), errno); + return 1; + } + } + else + { + HANDLE src, dst{sink.stream}; + for (;;) + { + src= CreateFile(src_path.c_str(), GENERIC_READ, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (src != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + + my_osmaperr(GetLastError()); + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), src_path.c_str(), + errno); + return -1; + } + + LARGE_INTEGER li; + if (!GetFileSizeEx(src, &li)) + { + write_error: + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); + if (src != INVALID_HANDLE_VALUE) + CloseHandle(src); + return -1; + } + + if (backup_stream_start(dst, path, 0644, li.QuadPart, nullptr, 0) || + backup_stream_append_plain(src, dst, 0, li.QuadPart)) + goto write_error; + + if (size_t pad= size_t(li.LowPart) & 511) + if (backup_stream_write(dst, zerobuf, 512 - pad)) + goto write_error; + if (!CloseHandle(src)) + { + src= INVALID_HANDLE_VALUE; + goto write_error; + } + } + return 0; +#endif + } + + + static bool is_db_file(const char* file_name) noexcept + { + size_t len= strlen(file_name); + if (len < 4) + return false; + uint32_t suffix; + memcpy(&suffix, file_name + len - 4, 4); + switch (suffix) { + default: + return len == 6 && !memcmp(file_name, C_STRING_WITH_LEN("db.opt")); +#ifdef WORDS_BIGENDIAN + case 0x2e41524d: /* .ARM ENGINE=ARCHIVE metadata */ + case 0x2e41525a: /* .ARZ ENGINE=ARCHIVE compressed data */ + case 0x2e43534d: /* .CSM ENGINE=CSV metadata */ + case 0x2e435356: /* .CSV ENGINE=CSV data ("comma separated values") */ + case 0x2e4d4144: /* .MAD ENGINE=Aria data heap */ + case 0x2e4d4149: /* .MAI ENGINE=Aria indexes */ + case 0x2e4d5247: /* .MRG ENGINE=MRG_MyISAM */ + case 0x2e4d5944: /* .MYD ENGINE=MyISAM data heap */ + case 0x2e4d5949: /* .MYI ENGINE=MyISAM indexes */ + case 0x2e66726d: /* .frm form (SHOW CREATE TABLE) */ + case 0x2e706172: /* .par PARTITION metadata */ +#else + case 0x4d52412e: /* .ARM ENGINE=ARCHIVE metadata */ + case 0x5a52412e: /* .ARZ ENGINE=ARCHIVE compressed data */ + case 0x4d53432e: /* .CSM ENGINE=CSV metadata */ + case 0x5653432e: /* .CSV ENGINE=CSV data ("comma separated values") */ + case 0x44414d2e: /* .MAD ENGINE=Aria data heap */ + case 0x49414d2e: /* .MAI ENGINE=Aria indexes */ + case 0x47524d2e: /* .MRG ENGINE=MRG_MyISAM */ + case 0x44594d2e: /* .MYD ENGINE=MyISAM data heap */ + case 0x49594d2e: /* .MYI ENGINE=MyISAM indexes */ + case 0x6d72662e: /* .frm form (SHOW CREATE TABLE) */ + case 0x7261702e: /* .par PARTITION metadata */ +#endif + return true; + } + } + +#ifdef _WIN32 + /** @return the target directory path */ + static std::string make_path(const backup_target &target, const char *name) + { + return std::string(target.path); + } +#endif + }; +} + +void *aria_backup_start(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + case BACKUP_PHASE_PREPARE_START: + return 0; + default: + return sink->ha_data; + case BACKUP_PHASE_NO_COMMIT: + assert(!sink->ha_data); + Aria_backup *aria_backup{new Aria_backup}; + if (aria_backup->initialize()) + { + delete aria_backup; + return reinterpret_cast(-1); + } + return aria_backup; + } +} + +#if 0 // FIXME: implement the actual copying here +int aria_backup_step(THD*, const backup_target*, backup_phase, + const backup_sink*) noexcept +{ + return 0; +} +#endif + +int aria_backup_end(THD *thd, const backup_target *target, backup_phase phase, + const backup_sink *sink) noexcept +{ + Aria_backup *aria_backup= static_cast(sink->ha_data); + switch (phase) { + case BACKUP_PHASE_NO_COMMIT: + assert(aria_backup); +#if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others + tc_purge(); + tdc_purge(true); +#endif + return aria_backup->end(*target, *sink); + case BACKUP_PHASE_FINISH: + delete aria_backup; + /* fall through */ + default: + return 0; + } +} diff --git a/storage/maria/ma_backup_server.h b/storage/maria/ma_backup_server.h new file mode 100644 index 0000000000000..6c72bb46eb60f --- /dev/null +++ b/storage/maria/ma_backup_server.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#pragma once + +/* BACKUP SERVER support for Aria engine. */ + +#include +#include + +/** + Start of a BACKUP SERVER phase, + when no aria_backup_step() or aria_backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to backup_target, or nullptr + @retval -1 on failure +*/ +void *aria_backup_start(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept; + +/** + Process a file that was collected in aria_backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @retval 0 on completion +*/ +int aria_backup_step(THD *thd, const backup_target *target, backup_phase phase, + const backup_sink *sink) noexcept; + +/** + Finish a phase, once all calls for the current phase are completed. + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked, + or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success +*/ +int aria_backup_end(THD *thd, const backup_target *target, backup_phase phase, + const backup_sink *sink) noexcept; From 49f4a3e51997c7f168a714cd86a6f596e6fad954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 29 Jun 2026 13:51:22 +0300 Subject: [PATCH 2/3] fixup! 81b3ae71537ca4c67ea4d0f740778f1596fd29a8 --- storage/maria/ma_backup_server.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/storage/maria/ma_backup_server.cc b/storage/maria/ma_backup_server.cc index 43c473a50a7ca..bac9989fd5ba1 100644 --- a/storage/maria/ma_backup_server.cc +++ b/storage/maria/ma_backup_server.cc @@ -368,7 +368,10 @@ namespace /** @return the target directory path */ static std::string make_path(const backup_target &target, const char *name) { - return std::string(target.path); + std::string path{target.path}; + path.push_back('/'); + path.append(name); + return path; } #endif }; From 6c59b142f4acb4a5a7477358a25c2a56ec915386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 29 Jun 2026 17:17:16 +0300 Subject: [PATCH 3/3] squash! 81b3ae71537ca4c67ea4d0f740778f1596fd29a8 Observe aria_log_dir_path Patch based on code by Thirunarayanan Balathandayuthapani --- storage/maria/ma_backup_server.cc | 87 ++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 30 deletions(-) diff --git a/storage/maria/ma_backup_server.cc b/storage/maria/ma_backup_server.cc index bac9989fd5ba1..9991875a6d2ac 100644 --- a/storage/maria/ma_backup_server.cc +++ b/storage/maria/ma_backup_server.cc @@ -43,6 +43,8 @@ namespace #ifndef _WIN32 if (datadir_fd >= 0) std::ignore= close(datadir_fd); + if (logdir_fd >= 0) + std::ignore= close(logdir_fd); #endif if (translog_purge_disabled) translog_enable_purge(); @@ -51,8 +53,18 @@ namespace bool initialize() noexcept { #ifndef _WIN32 - datadir_fd= open(maria_data_root, O_DIRECTORY); + /* Aria table files live under the server data directory + (mysql_real_data_home), while the transaction logs and control file + live under aria_log_dir_path (maria_data_root). These differ when + aria_log_dir_path is set, so open and scan them separately. */ + datadir_fd= open(mysql_real_data_home, O_DIRECTORY); if (datadir_fd < 0) + { + my_error(ER_CANT_READ_DIR, MYF(0), mysql_real_data_home, errno); + return true; + } + logdir_fd= open(maria_data_root, O_DIRECTORY); + if (logdir_fd < 0) { my_error(ER_CANT_READ_DIR, MYF(0), maria_data_root, errno); return true; @@ -74,8 +86,10 @@ namespace } private: #ifndef _WIN32 - /** The server data directory */ + /** The server data directory (Aria table files) */ int datadir_fd{-1}; + /** The Aria log directory aria_log_dir_path (logs, control file) */ + int logdir_fd{-1}; #endif /** whether the Aria translog_disable_purge() is in effect */ bool translog_purge_disabled{false}; @@ -104,32 +118,41 @@ namespace int scan_datadir() noexcept { - const char *base_dir= maria_data_root; - MY_DIR *dir_info= my_dir(base_dir, MYF(MY_WANT_STAT)); - if (!dir_info) - return dir_error(base_dir); + /* Scan the server data directory for Aria table files. */ + MY_DIR *data_dir= my_dir(mysql_real_data_home, MYF(MY_WANT_STAT)); + if (!data_dir) + return dir_error(mysql_real_data_home); int fail= 0; for (const fileinfo &fi : - st_::span{dir_info->dir_entry, - dir_info->number_of_files}) + st_::span{data_dir->dir_entry, + data_dir->number_of_files}) if ((fi.mystat->st_mode & S_IFMT) == S_IFDIR) - { if ((fail= scan_database_dir(fi.name)) != 0) - goto func_exit; - } - else if (!strncmp(fi.name, C_STRING_WITH_LEN("aria_log."))) + break; + my_dirend(data_dir); + if (fail) + return fail; + + /* Scan aria_log_dir_path for the transaction logs and control file. */ + MY_DIR *log_dir= my_dir(maria_data_root, MYF(MY_WANT_STAT)); + if (!log_dir) + return dir_error(maria_data_root); + for (const fileinfo &fi : + st_::span{log_dir->dir_entry, + log_dir->number_of_files}) + { + if (!strncmp(fi.name, C_STRING_WITH_LEN("aria_log."))) log_files.emplace_back(fi.name); else if (!strcmp(fi.name, "aria_log_control")) have_control_file = true; - func_exit: - my_dirend(dir_info); - return fail; + } + my_dirend(log_dir); + return 0; } int scan_database_dir(const char* dir_name) noexcept { - const char* base_dir = maria_data_root; - std::string dir_path = std::string(base_dir) + "/" + dir_name; + const std::string dir_path{make_path(mysql_real_data_home, dir_name)}; MY_DIR *dir_info= my_dir(dir_path.c_str(), MYF(MY_WANT_STAT)); if (!dir_info) return dir_error(dir_path.c_str()); @@ -167,7 +190,7 @@ namespace noexcept { #ifdef _WIN32 - if (CreateDirectory(make_path(target, name).c_str(), nullptr)) + if (CreateDirectory(make_path(target.dir, name).c_str(), nullptr)) return 0; DWORD err= GetLastError(); if (err == ERROR_ALREADY_EXISTS) @@ -190,7 +213,7 @@ namespace file_path= dir.first; file_path.push_back('/'); file_path.append(file); - if (int fail= copy_file(target, sink, file_path.c_str())) + if (int fail= copy_file(target, sink, file_path.c_str(), false)) return fail; } return 0; @@ -201,24 +224,24 @@ namespace { if (!have_control_file) return 0; - return copy_file(target, sink, "aria_log_control"); + return copy_file(target, sink, "aria_log_control", true); } int copy_logs(const backup_target &target, const backup_sink &sink) noexcept { for (const std::string &file : log_files) - if (int fail= copy_file(target, sink, file.c_str())) + if (int fail= copy_file(target, sink, file.c_str(), true)) return fail; return 0; } int copy_file(const backup_target &target, const backup_sink &sink, - const char *path) const noexcept + const char *path, bool is_log) const noexcept { #ifndef _WIN32 int ret_val{0}; - int src_fd{openat(datadir_fd, path, O_RDONLY)}; + int src_fd{openat(is_log ? logdir_fd : datadir_fd, path, O_RDONLY)}; if (src_fd < 0) { my_error(ER_CANT_OPEN_FILE, MYF(0), path, errno); @@ -259,11 +282,12 @@ namespace close(src_fd); return ret_val; #else - std::string src_path= std::string(maria_data_root) + "/" + path; + const std::string src_path + {make_path(is_log ? maria_data_root : mysql_real_data_home, path)}; if (sink.stream == sink.NO_STREAM) { - std::string dest_path{make_path(target, path)}; + std::string dest_path{make_path(target.dir, path)}; if (!CopyFileEx(src_path.c_str(), dest_path.c_str(), nullptr, nullptr, nullptr, COPY_FILE_NO_BUFFERING)) { @@ -364,16 +388,19 @@ namespace } } -#ifdef _WIN32 - /** @return the target directory path */ - static std::string make_path(const backup_target &target, const char *name) + /** + Construct a file path. + @param dir directory name + @param name file name + @return dir/name + */ + static std::string make_path(const char *dir, const char *name) { - std::string path{target.path}; + std::string path{dir}; path.push_back('/'); path.append(name); return path; } -#endif }; }