MDEV-22186 : Add innodb_buffer_pool_in_core_file to control buffer pool in core

mysql-test/include/linux-version.inc

@@ -0,0 +1,53 @@
+# Verifies that we are on linux, and that version returned by uname -r is
+# at least as large as $minimum_required_linux_version.
+# Otherwise it skips the test.
+
+--source 'include/linux.inc'
+
+if (!$minimum_required_linux_version){
+  --die You must specify $minimum_required_linux_version
+}
+
+let MINIMUM_REQUIRED_LINUX_VERSION_FOR_PERL = $minimum_required_linux_version;
+let LINUX_VERSION_RESULT_FILE = $MYSQLTEST_VARDIR/log/linux_version_result.inc;
+
+perl;
+  use strict;
+
+  # To keep it simple we only look at major and minor parts of uname -r,
+  # as later parts may contain non-digits and it is not clear how we
+  # should handle that.
+  # Don't be tempted to use $Config{osvers} here, as there are machines
+  # on which `uname` correctly reports 2.6 and $Config reports 3.8.
+  my $version= (split /\n/, `uname -r | cut -d '.' -f 1-2`)[0];
+  my $minimum_required_version= $ENV{'MINIMUM_REQUIRED_LINUX_VERSION_FOR_PERL'};
+
+  # Retrieving result from --perl command is non-trivial as of today, so
+  # we need to create an *.inc file on the fly, that will contain result.
+  open(RESULT_FILE, ">$ENV{'LINUX_VERSION_RESULT_FILE'}");
+
+  # This would be much easier using CPAN::Version, but we can't rely on it
+  # being avialable in the environment. This simple implementation cares only
+  # about major and minor numbers. It was tested for 4 < 4.1, 4.1 < 4.10,
+  # 4.12 < 4.111 and 3.4 < 4.1 (and their mirror images).
+
+  my @version_parts = split /\./, $version;
+  my @minimum_required_version_parts = split /\./, $minimum_required_version;
+
+  if ((@version_parts[0] <=> @minimum_required_version_parts[0] ||
+       @version_parts[1] <=> @minimum_required_version_parts[1]) < 0) {
+    print RESULT_FILE "let \$linux_version_is_ok = 0;\n";
+    print RESULT_FILE "let \$found_linux_version = $version;\n";
+  } else {
+    print RESULT_FILE "let \$linux_version_is_ok = 1;\n";
+  }
+  close(RESULT_FILE);
+EOF
+
+--source $LINUX_VERSION_RESULT_FILE
+--remove_file $LINUX_VERSION_RESULT_FILE
+
+if (!$linux_version_is_ok)
+{
+  skip Needs Linux $minimum_required_linux_version, found $found_linux_version;
+}

mysql-test/include/mysqld_core_dump.inc

@@ -0,0 +1,97 @@
+--echo # Get the full path name of the PID file
+--let $pid_file= query_get_value(SELECT @@pid_file, @@pid_file, 1)
+--let PIDFILE= $pid_file
+
+--echo # Expecting a "crash", but don't restart the server until it is told to
+--echo # Expected max core size is $expected_max_core_size MB
+--let MAXCORESIZE= $expected_max_core_size
+
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+
+perl;
+
+my $pid_file = $ENV{'PIDFILE'} or die "PIDFILE not set";
+my $expected_max_core_size = $ENV{'MAXCORESIZE'} or die "MAXCORESIZE not set";
+
+# The argument is in MB
+$expected_max_core_size = $expected_max_core_size * 1024 * 1024;
+
+# Get PID of mysqld
+open(my $fh, '<', $pid_file) || die "Cannot open pid file $pid_file\n";
+my $pid = <$fh>;
+$pid =~ s/\s//g;
+close($fh);
+
+if ($pid eq "") {
+  die "Couldn't retrieve PID from PID file.\n";
+}
+
+# The current time in seconds since epoch
+$cur_time = time;
+
+# Kill mysqld to dump a core
+system("kill", "-s", "SIGABRT", "$pid");
+print "# Perl: Sent a SIGABRT to mysqld to dump a core.\n";
+
+$core_dir = $ENV{'MYSQLTEST_VARDIR'} . '/mysqld.1/data/';
+
+$found_core = 0;
+$core_size = 0;
+$core_size_good = 0;
+
+# Check the files in the core file directory
+$wait_sec = 60;
+while ($wait_sec > 0) {
+  opendir(my $dir, $core_dir) or die "Failed to open dir $core_dir: $!\n";
+  while (my $file = readdir($dir)) {
+    # If the core file name contains the PID or is 'core'
+    if (index($file, $pid) != -1 || $file eq 'core') {
+      # The last write time in seconds since epoch
+      $full_path = $core_dir . '/' . $file;
+      @stat = stat($full_path);
+      $core_size = $stat[7];
+      $write_secs = $stat[9];
+
+      # If the file was written within a minute
+      if ($cur_time <= $write_secs && $write_secs - $cur_time < 60) {
+        $found_core = 1;
+        if ($core_size < $expected_max_core_size) {
+          $core_size_good = 1;
+        }
+        # Remove the core file to avoid it get accumulated over time
+        unlink $full_path;
+        last;
+      }
+    }
+  }
+  closedir($dir);
+
+  if ($found_core) {
+    last;
+  }
+  # Sleep 1 second and try again
+  --$wait_sec;
+  sleep 1;
+}
+
+if ($found_core) {
+  if ($core_size_good) {
+    print "# Perl: OK! Found the core file and it's small!\n";
+  } else {
+    print "# Perl: Failed! Found the core file but it's too big ($core_size)!\n";
+  }
+} else {
+  print "# Perl: Failed! Didn't find the core file!\n";
+}
+
+EOF
+
+--echo # Make server restart
+--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+
+--enable_reconnect
+
+--echo # Wait for server to be back online
+--source include/wait_until_connected_again.inc
+
+--disable_reconnect

mysql-test/suite/innodb/r/mysqld_core_dump_without_buffer_pool.result

@@ -0,0 +1,13 @@
+# Shutdown server
+# Restart server with --log-error
+SHOW VARIABLES LIKE '%core%';
+Variable_name	Value
+core_file	ON
+innodb_buffer_pool_in_core_file	OFF
+# Get the full path name of the PID file
+# Expecting a "crash", but don't restart the server until it is told to
+# Expected max core size is 1024 MB
+# Perl: Sent a SIGABRT to mysqld to dump a core.
+# Perl: OK! Found the core file and it's small!
+# Make server restart
+# Wait for server to be back online

mysql-test/suite/innodb/r/mysqld_core_dump_without_buffer_pool_dynamic.result

@@ -0,0 +1,28 @@
+# Shutdown server
+# Restart server with --log-error
+SELECT @@global.innodb_buffer_pool_in_core_file;
+@@global.innodb_buffer_pool_in_core_file
+1
+SET GLOBAL innodb_buffer_pool_in_core_file = OFF;
+SELECT @@global.innodb_buffer_pool_in_core_file;
+@@global.innodb_buffer_pool_in_core_file
+0
+SET GLOBAL innodb_buffer_pool_in_core_file = ON;
+SELECT @@global.innodb_buffer_pool_in_core_file;
+@@global.innodb_buffer_pool_in_core_file
+1
+SET GLOBAL innodb_buffer_pool_in_core_file = OFF;
+SELECT @@global.innodb_buffer_pool_in_core_file;
+@@global.innodb_buffer_pool_in_core_file
+0
+SHOW VARIABLES LIKE '%core%';
+Variable_name	Value
+core_file	ON
+innodb_buffer_pool_in_core_file	OFF
+# Get the full path name of the PID file
+# Expecting a "crash", but don't restart the server until it is told to
+# Expected max core size is 1024 MB
+# Perl: Sent a SIGABRT to mysqld to dump a core.
+# Perl: OK! Found the core file and it's small!
+# Make server restart
+# Wait for server to be back online

mysql-test/suite/innodb/r/mysqld_core_dump_without_buffer_pool_with_resizing.result

@@ -0,0 +1,15 @@
+# Shutdown server
+# Restart server with --log-error
+set global innodb_buffer_pool_size = 512*1024*1024;
+set global innodb_buffer_pool_size = 2048*1024*1024;
+SHOW VARIABLES LIKE '%core%';
+Variable_name	Value
+core_file	ON
+innodb_buffer_pool_in_core_file	OFF
+# Get the full path name of the PID file
+# Expecting a "crash", but don't restart the server until it is told to
+# Expected max core size is 1024 MB
+# Perl: Sent a SIGABRT to mysqld to dump a core.
+# Perl: OK! Found the core file and it's small!
+# Make server restart
+# Wait for server to be back online

mysql-test/suite/innodb/t/mysqld_core_dump_without_buffer_pool-master.opt

@@ -0,0 +1,3 @@
+--core-file
+--skip-innodb-buffer-pool-in-core-file
+--innodb-buffer-pool-size=1G

mysql-test/suite/innodb/t/mysqld_core_dump_without_buffer_pool.test

@@ -0,0 +1,29 @@
+################################################################################
+# This test is to test if mysqld can dump a core without large memory buffers.
+# See opt file for the config:
+#   (1) --skip-innodb-buffer-pool-in-core-file is set
+#   (2) the buffer pool is set to be 1G so that with the large
+#       memory buffers the core size would be much greater than 1GB (the actual
+#       core size observed is ~1.8GB) and without them is less then 1GB (the
+#       actual observed size is ~700MB)
+
+# madvise MADV_DONTDUMP is non-posix extension available in Linux 3.4
+--let $minimum_required_linux_version = 3.4
+--source include/linux-version.inc
+--source include/have_innodb.inc
+--source include/not_valgrind.inc
+
+# Embedded mode doesn't support restart
+--source include/not_embedded.inc
+
+--echo # Shutdown server
+--source include/shutdown_mysqld.inc
+
+--echo # Restart server with --log-error
+--exec echo "restart:--log-error=$MYSQLTEST_VARDIR/log/core_dump.err" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+SHOW VARIABLES LIKE '%core%';
+--let $expected_max_core_size = 1024
+--source include/mysqld_core_dump.inc

mysql-test/suite/innodb/t/mysqld_core_dump_without_buffer_pool_dynamic-master.opt

@@ -0,0 +1,3 @@
+--core-file
+--innodb-buffer-pool-in-core-file
+--innodb-buffer-pool-size=1G

mysql-test/suite/innodb/t/mysqld_core_dump_without_buffer_pool_dynamic.test

@@ -0,0 +1,42 @@
+################################################################################
+# This test is to test if mysqld can dump a core without large memory buffers.
+# We start the server with --innodb-buffer-pool-in-core-file ON, then
+# we dynamically set it to OFF.
+# Just to play with possible race conditions we switch it to ON and OFF one
+# more time.
+# Finally we cause core dump to happen.
+# See opt file for the config:
+#   (1) --innodb-buffer-pool-in-core-file is set to ON initially
+#   (2) the buffer pool is set to be 1G so that with the large
+#       memory buffers the core size would be much greater than 1GB (the actual
+#       core size observed is ~1.8GB) and without them is less then 1GB (the
+#       actual observed size is ~700MB)
+
+# madvise MADV_DONTDUMP is non-posix extension available in Linux 3.4
+--let $minimum_required_linux_version = 3.4
+--source include/linux-version.inc
+--source include/have_innodb.inc
+--source include/not_valgrind.inc
+
+# Embedded mode doesn't support restart
+--source include/not_embedded.inc
+
+--echo # Shutdown server
+--source include/shutdown_mysqld.inc
+
+--echo # Restart server with --log-error
+--exec echo "restart:--log-error=$MYSQLTEST_VARDIR/log/core_dump.err" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+SELECT @@global.innodb_buffer_pool_in_core_file;
+SET GLOBAL innodb_buffer_pool_in_core_file = OFF;
+SELECT @@global.innodb_buffer_pool_in_core_file;
+SET GLOBAL innodb_buffer_pool_in_core_file = ON;
+SELECT @@global.innodb_buffer_pool_in_core_file;
+SET GLOBAL innodb_buffer_pool_in_core_file = OFF;
+SELECT @@global.innodb_buffer_pool_in_core_file;
+SHOW VARIABLES LIKE '%core%';
+
+--let $expected_max_core_size = 1024
+--source include/mysqld_core_dump.inc

mysql-test/suite/innodb/t/mysqld_core_dump_without_buffer_pool_with_resizing-master.opt

@@ -0,0 +1,5 @@
+--core-file
+--skip-innodb-buffer-pool-in-core-file
+--innodb-buffer-pool-size-auto-min=256M
+--innodb-buffer-pool-size=1280M
+--innodb-buffer-pool-size-max=2G

mysql-test/suite/innodb/t/mysqld_core_dump_without_buffer_pool_with_resizing.test

@@ -0,0 +1,46 @@
+################################################################################
+# This test is to test if mysqld can dump a core without large memory buffers.
+# See opt file for the config:
+#   (1) --skip-innodb-buffer-pool-in-core-file is set
+#   (2) the buffer pool is set to be 1280MB initially, shrink it to 1024MB, then
+#       expand it back to 2048MB, then back to original 1280MB.
+#       With the large memory buffers the core size will be much greater than 1GB
+#       (the actual observed size is ~2.2.G) and without them it will be smaller
+#       than 1GB (actually ~770MB)
+
+# madvise MADV_DONTDUMP is non-posix extension available in Linux 3.4
+--let $minimum_required_linux_version = 3.4
+--source include/linux-version.inc
+--source include/have_innodb.inc
+--source include/not_valgrind.inc
+
+# Embedded mode doesn't support restart
+--source include/not_embedded.inc
+
+--echo # Shutdown server
+--source include/shutdown_mysqld.inc
+
+--echo # Restart server with --log-error
+--exec echo "restart:--log-error=$MYSQLTEST_VARDIR/log/core_dump_with_resizing.err" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+--disable_query_log
+set @old_innodb_buffer_pool_size = @@innodb_buffer_pool_size;
+--enable_query_log
+
+# Shrink buffer pool to 500MB
+set global innodb_buffer_pool_size = 512*1024*1024;
+
+# Expand buffer pool back to 2GB
+set global innodb_buffer_pool_size = 2048*1024*1024;
+
+# Resize buffer pool back to original 1280MB
+--disable_query_log
+set global innodb_buffer_pool_size = @old_innodb_buffer_pool_size;
+--enable_query_log
+
+SHOW VARIABLES LIKE '%core%';
+
+--let $expected_max_core_size = 1024
+--source include/mysqld_core_dump.inc

mysql-test/suite/sys_vars/r/sysvars_innodb.result

@@ -164,6 +164,18 @@ NUMERIC_BLOCK_SIZE	NULL
 ENUM_VALUE_LIST	NULL
 READ_ONLY	YES
 COMMAND_LINE_ARGUMENT	REQUIRED
+VARIABLE_NAME	INNODB_BUFFER_POOL_IN_CORE_FILE
+SESSION_VALUE	NULL
+DEFAULT_VALUE	ON
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	BOOLEAN
+VARIABLE_COMMENT	This option has no effect if @@core_file is OFF. If @@core_file is ON, and this option is OFF, then the core dump file will be generated only if it is possible to exclude buffer pool from it. As soon as it will be determined that such exclusion is impossible a warning will be emitted and @@core_file will be set to OFF to prevent generating a core dump. This option is disabled by default if the build is not a debug build and platform supports MADV_DONTDUMP.
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	OFF,ON
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	NONE
 VARIABLE_NAME	INNODB_BUFFER_POOL_LOAD_ABORT
 SESSION_VALUE	NULL
 DEFAULT_VALUE	OFF