nixos/tests/mariadb-galera: test multiple mariadb versions

merge both tests and completely refactor

5 files changed, 261 insertions, 461 deletions

diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 940ae11ddd1..93950277c27 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -268,8 +268,7 @@ in
   mailcatcher = handleTest ./mailcatcher.nix {};
   mailhog = handleTest ./mailhog.nix {};
   man = handleTest ./man.nix {};
-  mariadb-galera-mariabackup = handleTest ./mysql/mariadb-galera-mariabackup.nix {};
-  mariadb-galera-rsync = handleTest ./mysql/mariadb-galera-rsync.nix {};
+  mariadb-galera = handleTest ./mysql/mariadb-galera.nix {};
   matomo = handleTest ./matomo.nix {};
   matrix-appservice-irc = handleTest ./matrix-appservice-irc.nix {};
   matrix-conduit = handleTest ./matrix-conduit.nix {};
diff --git a/nixos/tests/mysql/common.nix b/nixos/tests/mysql/common.nix
new file mode 100644
index 00000000000..968253d109c
--- /dev/null
+++ b/nixos/tests/mysql/common.nix
@@ -0,0 +1,10 @@
+{ lib, pkgs }: {
+  mariadbPackages = lib.filterAttrs (n: _: lib.hasPrefix "mariadb" n) (pkgs.callPackage ../../../pkgs/servers/sql/mariadb {
+    inherit (pkgs.darwin) cctools;
+    inherit (pkgs.darwin.apple_sdk.frameworks) CoreServices;
+  });
+  mysqlPackage = {
+    inherit (pkgs) mysql57 mysql80;
+  };
+  mkTestName = pkg: "mariadb_${builtins.replaceStrings ["."] [""] (lib.versions.majorMinor pkg.version)}";
+}
diff --git a/nixos/tests/mysql/mariadb-galera-mariabackup.nix b/nixos/tests/mysql/mariadb-galera-mariabackup.nix
deleted file mode 100644
index 10682c361d1..00000000000
--- a/nixos/tests/mysql/mariadb-galera-mariabackup.nix
+++ /dev/null
@@ -1,233 +0,0 @@
-import ./../make-test-python.nix ({ pkgs, ...} :
-
-let
-  mysqlenv-common      = pkgs.buildEnv { name = "mysql-path-env-common";      pathsToLink = [ "/bin" ]; paths = with pkgs; [ bash gawk gnutar inetutils which ]; };
-  mysqlenv-mariabackup = pkgs.buildEnv { name = "mysql-path-env-mariabackup"; pathsToLink = [ "/bin" ]; paths = with pkgs; [ gzip iproute2 netcat procps pv socat ]; };
-
-  # Common user configuration
-  users = { ... }:
-  {
-    users.users.testuser = {
-      isSystemUser = true;
-      group = "testusers";
-    };
-    users.groups.testusers = { };
-  };
-
-in {
-  name = "mariadb-galera-mariabackup";
-  meta = with pkgs.lib.maintainers; {
-    maintainers = [ izorkin ];
-  };
-
-  # The test creates a Galera cluster with 3 nodes and is checking if mariabackup-based SST works. The cluster is tested by creating a DB and an empty table on one node,
-  # and checking the table's presence on the other node.
-
-  nodes = {
-    galera_01 =
-      { pkgs, ... }:
-      {
-      imports = [ users ];
-      networking = {
-        interfaces.eth1 = {
-          ipv4.addresses = [
-            { address = "192.168.1.1"; prefixLength = 24; }
-          ];
-        };
-        extraHosts = ''
-          192.168.1.1 galera_01
-          192.168.1.2 galera_02
-          192.168.1.3 galera_03
-        '';
-        firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
-        firewall.allowedUDPPorts = [ 4567 ];
-      };
-      systemd.services.mysql = with pkgs; {
-        path = [ mysqlenv-common mysqlenv-mariabackup ];
-      };
-      services.mysql = {
-        enable = true;
-        package = pkgs.mariadb;
-        ensureDatabases = [ "testdb" ];
-        ensureUsers = [{
-          name = "testuser";
-          ensurePermissions = {
-            "testdb.*" = "ALL PRIVILEGES";
-          };
-        }];
-        initialScript = pkgs.writeText "mariadb-init.sql" ''
-          GRANT ALL PRIVILEGES ON *.* TO 'check_repl'@'localhost' IDENTIFIED BY 'check_pass' WITH GRANT OPTION;
-          FLUSH PRIVILEGES;
-        '';
-        settings = {
-          mysqld = {
-            bind_address = "0.0.0.0";
-          };
-          galera = {
-            wsrep_on = "ON";
-            wsrep_debug = "NONE";
-            wsrep_retry_autocommit = "3";
-            wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
-            wsrep_cluster_address = "gcomm://";
-            wsrep_cluster_name = "galera";
-            wsrep_node_address = "192.168.1.1";
-            wsrep_node_name = "galera_01";
-            wsrep_sst_method = "mariabackup";
-            wsrep_sst_auth = "check_repl:check_pass";
-            binlog_format = "ROW";
-            enforce_storage_engine = "InnoDB";
-            innodb_autoinc_lock_mode = "2";
-          };
-        };
-      };
-    };
-
-    galera_02 =
-      { pkgs, ... }:
-      {
-      imports = [ users ];
-      networking = {
-        interfaces.eth1 = {
-          ipv4.addresses = [
-            { address = "192.168.1.2"; prefixLength = 24; }
-          ];
-        };
-        extraHosts = ''
-          192.168.1.1 galera_01
-          192.168.1.2 galera_02
-          192.168.1.3 galera_03
-        '';
-        firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
-        firewall.allowedUDPPorts = [ 4567 ];
-      };
-      systemd.services.mysql = with pkgs; {
-        path = [ mysqlenv-common mysqlenv-mariabackup ];
-      };
-      services.mysql = {
-        enable = true;
-        package = pkgs.mariadb;
-        settings = {
-          mysqld = {
-            bind_address = "0.0.0.0";
-          };
-          galera = {
-            wsrep_on = "ON";
-            wsrep_debug = "NONE";
-            wsrep_retry_autocommit = "3";
-            wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
-            wsrep_cluster_address = "gcomm://galera_01,galera_02,galera_03";
-            wsrep_cluster_name = "galera";
-            wsrep_node_address = "192.168.1.2";
-            wsrep_node_name = "galera_02";
-            wsrep_sst_method = "mariabackup";
-            wsrep_sst_auth = "check_repl:check_pass";
-            binlog_format = "ROW";
-            enforce_storage_engine = "InnoDB";
-            innodb_autoinc_lock_mode = "2";
-          };
-        };
-      };
-    };
-
-    galera_03 =
-      { pkgs, ... }:
-      {
-      imports = [ users ];
-      networking = {
-        interfaces.eth1 = {
-          ipv4.addresses = [
-            { address = "192.168.1.3"; prefixLength = 24; }
-          ];
-        };
-        extraHosts = ''
-          192.168.1.1 galera_01
-          192.168.1.2 galera_02
-          192.168.1.3 galera_03
-        '';
-        firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
-        firewall.allowedUDPPorts = [ 4567 ];
-      };
-      systemd.services.mysql = with pkgs; {
-        path = [ mysqlenv-common mysqlenv-mariabackup ];
-      };
-      services.mysql = {
-        enable = true;
-        package = pkgs.mariadb;
-        settings = {
-          mysqld = {
-            bind_address = "0.0.0.0";
-          };
-          galera = {
-            wsrep_on = "ON";
-            wsrep_debug = "NONE";
-            wsrep_retry_autocommit = "3";
-            wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
-            wsrep_cluster_address = "gcomm://galera_01,galera_02,galera_03";
-            wsrep_cluster_name = "galera";
-            wsrep_node_address = "192.168.1.3";
-            wsrep_node_name = "galera_03";
-            wsrep_sst_method = "mariabackup";
-            wsrep_sst_auth = "check_repl:check_pass";
-            binlog_format = "ROW";
-            enforce_storage_engine = "InnoDB";
-            innodb_autoinc_lock_mode = "2";
-          };
-        };
-      };
-    };
-  };
-
-  testScript = ''
-    galera_01.start()
-    galera_01.wait_for_unit("mysql")
-    galera_01.wait_for_open_port(3306)
-    galera_01.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; create table db1 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
-    )
-    galera_01.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db1 values (37);'"
-    )
-    galera_02.start()
-    galera_02.wait_for_unit("mysql")
-    galera_02.wait_for_open_port(3306)
-    galera_03.start()
-    galera_03.wait_for_unit("mysql")
-    galera_03.wait_for_open_port(3306)
-    galera_02.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 37"
-    )
-    galera_02.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; create table db2 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
-    )
-    galera_02.succeed("systemctl stop mysql")
-    galera_01.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db2 values (38);'"
-    )
-    galera_03.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; create table db3 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
-    )
-    galera_01.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db3 values (39);'"
-    )
-    galera_02.succeed("systemctl start mysql")
-    galera_02.wait_for_open_port(3306)
-    galera_02.succeed(
-        "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_cluster_size.*3'"
-    )
-    galera_03.succeed(
-        "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_local_state_comment.*Synced'"
-    )
-    galera_01.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db3;' -N | grep 39"
-    )
-    galera_02.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db2;' -N | grep 38"
-    )
-    galera_03.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 37"
-    )
-    galera_01.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db3;'")
-    galera_02.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db2;'")
-    galera_03.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db1;'")
-  '';
-})
diff --git a/nixos/tests/mysql/mariadb-galera-rsync.nix b/nixos/tests/mysql/mariadb-galera-rsync.nix
deleted file mode 100644
index 701e01e8871..00000000000
--- a/nixos/tests/mysql/mariadb-galera-rsync.nix
+++ /dev/null
@@ -1,226 +0,0 @@
-import ./../make-test-python.nix ({ pkgs, ...} :
-
-let
-  mysqlenv-common      = pkgs.buildEnv { name = "mysql-path-env-common";      pathsToLink = [ "/bin" ]; paths = with pkgs; [ bash gawk gnutar inetutils which ]; };
-  mysqlenv-rsync       = pkgs.buildEnv { name = "mysql-path-env-rsync";       pathsToLink = [ "/bin" ]; paths = with pkgs; [ lsof procps rsync stunnel ]; };
-
-  # Common user configuration
-  users = { ... }:
-  {
-    users.users.testuser = {
-      isSystemUser = true;
-      group = "testusers";
-    };
-    users.groups.testusers = { };
-  };
-
-in {
-  name = "mariadb-galera-rsync";
-  meta = with pkgs.lib.maintainers; {
-    maintainers = [ izorkin ];
-  };
-
-  # The test creates a Galera cluster with 3 nodes and is checking if rsync-based SST works. The cluster is tested by creating a DB and an empty table on one node,
-  # and checking the table's presence on the other node.
-
-  nodes = {
-    galera_04 =
-      { pkgs, ... }:
-      {
-      imports = [ users ];
-      networking = {
-        interfaces.eth1 = {
-          ipv4.addresses = [
-            { address = "192.168.2.1"; prefixLength = 24; }
-          ];
-        };
-        extraHosts = ''
-          192.168.2.1 galera_04
-          192.168.2.2 galera_05
-          192.168.2.3 galera_06
-        '';
-        firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
-        firewall.allowedUDPPorts = [ 4567 ];
-      };
-      systemd.services.mysql = with pkgs; {
-        path = [ mysqlenv-common mysqlenv-rsync ];
-      };
-      services.mysql = {
-        enable = true;
-        package = pkgs.mariadb;
-        ensureDatabases = [ "testdb" ];
-        ensureUsers = [{
-          name = "testuser";
-          ensurePermissions = {
-            "testdb.*" = "ALL PRIVILEGES";
-          };
-        }];
-        settings = {
-          mysqld = {
-            bind_address = "0.0.0.0";
-          };
-          galera = {
-            wsrep_on = "ON";
-            wsrep_debug = "NONE";
-            wsrep_retry_autocommit = "3";
-            wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
-            wsrep_cluster_address = "gcomm://";
-            wsrep_cluster_name = "galera-rsync";
-            wsrep_node_address = "192.168.2.1";
-            wsrep_node_name = "galera_04";
-            wsrep_sst_method = "rsync";
-            binlog_format = "ROW";
-            enforce_storage_engine = "InnoDB";
-            innodb_autoinc_lock_mode = "2";
-          };
-        };
-      };
-    };
-
-    galera_05 =
-      { pkgs, ... }:
-      {
-      imports = [ users ];
-      networking = {
-        interfaces.eth1 = {
-          ipv4.addresses = [
-            { address = "192.168.2.2"; prefixLength = 24; }
-          ];
-        };
-        extraHosts = ''
-          192.168.2.1 galera_04
-          192.168.2.2 galera_05
-          192.168.2.3 galera_06
-        '';
-        firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
-        firewall.allowedUDPPorts = [ 4567 ];
-      };
-      systemd.services.mysql = with pkgs; {
-        path = [ mysqlenv-common mysqlenv-rsync ];
-      };
-      services.mysql = {
-        enable = true;
-        package = pkgs.mariadb;
-        settings = {
-          mysqld = {
-            bind_address = "0.0.0.0";
-          };
-          galera = {
-            wsrep_on = "ON";
-            wsrep_debug = "NONE";
-            wsrep_retry_autocommit = "3";
-            wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
-            wsrep_cluster_address = "gcomm://galera_04,galera_05,galera_06";
-            wsrep_cluster_name = "galera-rsync";
-            wsrep_node_address = "192.168.2.2";
-            wsrep_node_name = "galera_05";
-            wsrep_sst_method = "rsync";
-            binlog_format = "ROW";
-            enforce_storage_engine = "InnoDB";
-            innodb_autoinc_lock_mode = "2";
-          };
-        };
-      };
-    };
-
-    galera_06 =
-      { pkgs, ... }:
-      {
-      imports = [ users ];
-      networking = {
-        interfaces.eth1 = {
-          ipv4.addresses = [
-            { address = "192.168.2.3"; prefixLength = 24; }
-          ];
-        };
-        extraHosts = ''
-          192.168.2.1 galera_04
-          192.168.2.2 galera_05
-          192.168.2.3 galera_06
-        '';
-        firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
-        firewall.allowedUDPPorts = [ 4567 ];
-      };
-      systemd.services.mysql = with pkgs; {
-        path = [ mysqlenv-common mysqlenv-rsync ];
-      };
-      services.mysql = {
-        enable = true;
-        package = pkgs.mariadb;
-        settings = {
-          mysqld = {
-            bind_address = "0.0.0.0";
-          };
-          galera = {
-            wsrep_on = "ON";
-            wsrep_debug = "NONE";
-            wsrep_retry_autocommit = "3";
-            wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
-            wsrep_cluster_address = "gcomm://galera_04,galera_05,galera_06";
-            wsrep_cluster_name = "galera-rsync";
-            wsrep_node_address = "192.168.2.3";
-            wsrep_node_name = "galera_06";
-            wsrep_sst_method = "rsync";
-            binlog_format = "ROW";
-            enforce_storage_engine = "InnoDB";
-            innodb_autoinc_lock_mode = "2";
-          };
-        };
-      };
-    };
-  };
-
-  testScript = ''
-    galera_04.start()
-    galera_04.wait_for_unit("mysql")
-    galera_04.wait_for_open_port(3306)
-    galera_04.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; create table db1 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
-    )
-    galera_04.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db1 values (41);'"
-    )
-    galera_05.start()
-    galera_05.wait_for_unit("mysql")
-    galera_05.wait_for_open_port(3306)
-    galera_06.start()
-    galera_06.wait_for_unit("mysql")
-    galera_06.wait_for_open_port(3306)
-    galera_05.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 41"
-    )
-    galera_05.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; create table db2 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
-    )
-    galera_05.succeed("systemctl stop mysql")
-    galera_04.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db2 values (42);'"
-    )
-    galera_06.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; create table db3 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
-    )
-    galera_04.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db3 values (43);'"
-    )
-    galera_05.succeed("systemctl start mysql")
-    galera_05.wait_for_open_port(3306)
-    galera_05.succeed(
-        "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_cluster_size.*3'"
-    )
-    galera_06.succeed(
-        "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_local_state_comment.*Synced'"
-    )
-    galera_04.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db3;' -N | grep 43"
-    )
-    galera_05.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db2;' -N | grep 42"
-    )
-    galera_06.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 41"
-    )
-    galera_04.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db3;'")
-    galera_05.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db2;'")
-    galera_06.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db1;'")
-  '';
-})
diff --git a/nixos/tests/mysql/mariadb-galera.nix b/nixos/tests/mysql/mariadb-galera.nix
new file mode 100644
index 00000000000..c9962f49c02
--- /dev/null
+++ b/nixos/tests/mysql/mariadb-galera.nix
@@ -0,0 +1,250 @@
+{
+  system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../../.. { inherit system config; },
+  lib ? pkgs.lib
+}:
+
+let
+  inherit (import ./common.nix { inherit pkgs lib; }) mkTestName mariadbPackages;
+
+  makeTest = import ./../make-test-python.nix;
+
+  # Common user configuration
+  makeGaleraTest = {
+    mariadbPackage,
+    name ? mkTestName mariadbPackage,
+    galeraPackage ? pkgs.mariadb-galera
+  }: makeTest {
+    name = "${name}-galera-mariabackup";
+    meta = with pkgs.lib.maintainers; {
+      maintainers = [ izorkin ajs124 das_j ];
+    };
+
+    # The test creates a Galera cluster with 3 nodes and is checking if mariabackup-based SST works. The cluster is tested by creating a DB and an empty table on one node,
+    # and checking the table's presence on the other node.
+    nodes = let
+      mkGaleraNode = {
+        id,
+        method
+      }: let
+        address = "192.168.1.${toString id}";
+        isFirstClusterNode = id == 1 || id == 4;
+      in {
+        users = {
+          users.testuser = {
+            isSystemUser = true;
+            group = "testusers";
+          };
+          groups.testusers = { };
+        };
+
+        networking = {
+          interfaces.eth1 = {
+            ipv4.addresses = [
+              { inherit address; prefixLength = 24; }
+            ];
+          };
+          extraHosts = lib.concatMapStringsSep "\n" (i: "192.168.1.${toString i} galera_0${toString i}") (lib.range 1 6);
+          firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
+          firewall.allowedUDPPorts = [ 4567 ];
+        };
+        systemd.services.mysql = with pkgs; {
+          path = with pkgs; [
+            bash
+            gawk
+            gnutar
+            gzip
+            inetutils
+            iproute2
+            netcat
+            procps
+            pv
+            rsync
+            socat
+            stunnel
+            which
+          ];
+        };
+        services.mysql = {
+          enable = true;
+          package = mariadbPackage;
+          ensureDatabases = lib.mkIf isFirstClusterNode [ "testdb" ];
+          ensureUsers = lib.mkIf isFirstClusterNode [{
+            name = "testuser";
+            ensurePermissions = {
+              "testdb.*" = "ALL PRIVILEGES";
+            };
+          }];
+          initialScript = lib.mkIf isFirstClusterNode (pkgs.writeText "mariadb-init.sql" ''
+            GRANT ALL PRIVILEGES ON *.* TO 'check_repl'@'localhost' IDENTIFIED BY 'check_pass' WITH GRANT OPTION;
+            FLUSH PRIVILEGES;
+          '');
+          settings = {
+            mysqld = {
+              bind_address = "0.0.0.0";
+            };
+            galera = {
+              wsrep_on = "ON";
+              wsrep_debug = "NONE";
+              wsrep_retry_autocommit = "3";
+              wsrep_provider = "${galeraPackage}/lib/galera/libgalera_smm.so";
+              wsrep_cluster_address = "gcomm://"
+                + lib.optionalString (id == 2 || id == 3) "galera_01,galera_02,galera_03"
+                + lib.optionalString (id == 5 || id == 6) "galera_04,galera_05,galera_06";
+              wsrep_cluster_name = "galera";
+              wsrep_node_address = address;
+              wsrep_node_name = "galera_0${toString id}";
+              wsrep_sst_method = method;
+              wsrep_sst_auth = "check_repl:check_pass";
+              binlog_format = "ROW";
+              enforce_storage_engine = "InnoDB";
+              innodb_autoinc_lock_mode = "2";
+            };
+          };
+        };
+      };
+    in {
+      galera_01 = mkGaleraNode {
+        id = 1;
+        method = "mariabackup";
+      };
+
+      galera_02 = mkGaleraNode {
+        id = 2;
+        method = "mariabackup";
+      };
+
+      galera_03 = mkGaleraNode {
+        id = 3;
+        method = "mariabackup";
+      };
+
+      galera_04 = mkGaleraNode {
+        id = 4;
+        method = "rsync";
+      };
+
+      galera_05 = mkGaleraNode {
+        id = 5;
+        method = "rsync";
+      };
+
+      galera_06 = mkGaleraNode {
+        id = 6;
+        method = "rsync";
+      };
+
+    };
+
+    testScript = ''
+      galera_01.start()
+      galera_01.wait_for_unit("mysql")
+      galera_01.wait_for_open_port(3306)
+      galera_01.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; create table db1 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
+      )
+      galera_01.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db1 values (37);'"
+      )
+      galera_02.start()
+      galera_02.wait_for_unit("mysql")
+      galera_02.wait_for_open_port(3306)
+      galera_03.start()
+      galera_03.wait_for_unit("mysql")
+      galera_03.wait_for_open_port(3306)
+      galera_02.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 37"
+      )
+      galera_02.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; create table db2 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
+      )
+      galera_02.succeed("systemctl stop mysql")
+      galera_01.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db2 values (38);'"
+      )
+      galera_03.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; create table db3 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
+      )
+      galera_01.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db3 values (39);'"
+      )
+      galera_02.succeed("systemctl start mysql")
+      galera_02.wait_for_open_port(3306)
+      galera_02.succeed(
+          "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_cluster_size.*3'"
+      )
+      galera_03.succeed(
+          "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_local_state_comment.*Synced'"
+      )
+      galera_01.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db3;' -N | grep 39"
+      )
+      galera_02.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db2;' -N | grep 38"
+      )
+      galera_03.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 37"
+      )
+      galera_01.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db3;'")
+      galera_02.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db2;'")
+      galera_03.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db1;'")
+      galera_01.crash()
+      galera_02.crash()
+      galera_03.crash()
+
+      galera_04.start()
+      galera_04.wait_for_unit("mysql")
+      galera_04.wait_for_open_port(3306)
+      galera_04.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; create table db1 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
+      )
+      galera_04.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db1 values (41);'"
+      )
+      galera_05.start()
+      galera_05.wait_for_unit("mysql")
+      galera_05.wait_for_open_port(3306)
+      galera_06.start()
+      galera_06.wait_for_unit("mysql")
+      galera_06.wait_for_open_port(3306)
+      galera_05.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 41"
+      )
+      galera_05.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; create table db2 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
+      )
+      galera_05.succeed("systemctl stop mysql")
+      galera_04.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db2 values (42);'"
+      )
+      galera_06.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; create table db3 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
+      )
+      galera_04.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db3 values (43);'"
+      )
+      galera_05.succeed("systemctl start mysql")
+      galera_05.wait_for_open_port(3306)
+      galera_05.succeed(
+          "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_cluster_size.*3'"
+      )
+      galera_06.succeed(
+          "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_local_state_comment.*Synced'"
+      )
+      galera_04.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db3;' -N | grep 43"
+      )
+      galera_05.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db2;' -N | grep 42"
+      )
+      galera_06.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 41"
+      )
+      galera_04.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db3;'")
+      galera_05.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db2;'")
+      galera_06.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db1;'")
+    '';
+  };
+in
+  lib.mapAttrs (_: mariadbPackage: makeGaleraTest { inherit mariadbPackage; }) mariadbPackages