nixos/systemd: migrate systemd-timesync state when required

Somewhen between systemd v239 and v242 upstream decided to no longer run
a few system services with `DyanmicUser=1` but failed to provide a
migration path for all the state those services left behind.

For the case of systemd-timesync the state has to be moved from
/var/lib/private/systemd/timesync to /var/lib/systemd/timesync if
/var/lib/systemd/timesync is currently a symlink.

We only do this if the stateVersion is still below 19.09 to avoid
starting to have an ever growing activation script for (then) ancient
systemd migrations that are no longer required.

See https://github.com/systemd/systemd/issues/12131 for details about
the missing migration path and related discussion.

5 files changed, 77 insertions, 3 deletions

diff --git a/nixos/doc/manual/release-notes/rl-1909.xml b/nixos/doc/manual/release-notes/rl-1909.xml
index bac642fa539..3b1a2d99794 100644
--- a/nixos/doc/manual/release-notes/rl-1909.xml
+++ b/nixos/doc/manual/release-notes/rl-1909.xml
@@ -172,6 +172,20 @@
       which is linked to <literal>fr-toutesvariantes.{aff,dic}</literal>.
     </para>
    </listitem>
-  </itemizedlist>
+   <listitem>
+    <para>
+      With the upgrade to systemd version 242 the <literal>systemd-timesyncd</literal>
+      service is no longer using <literal>DynamicUser=yes</literal>. In order for the
+      upgrade to work we rely on an activation script to move the state from the old
+      to the new directory. The older directory (prior <literal>19.09</literal>) was
+      <literal>/var/lib/private/systemd/timesync</literal>.
+    </para>
+    <para>
+      As long as the <literal>system.config.stateVersion</literal> is below
+      <literal>19.09</literal> the state folder will migrated to its proper location
+      (<literal>/var/lib/systemd/timesync</literal>), if required.
+    </para>
+  </listitem>
+ </itemizedlist>
  </section>
 </section>
diff --git a/nixos/modules/system/boot/systemd.nix b/nixos/modules/system/boot/systemd.nix
index 18ee2ef1b8f..8499b700d3e 100644
--- a/nixos/modules/system/boot/systemd.nix
+++ b/nixos/modules/system/boot/systemd.nix
@@ -940,7 +940,6 @@ in
     # Don't bother with certain units in containers.
     systemd.services.systemd-remount-fs.unitConfig.ConditionVirtualization = "!container";
     systemd.services.systemd-random-seed.unitConfig.ConditionVirtualization = "!container";
-
   };
 
   # FIXME: Remove these eventually.
@@ -949,5 +948,4 @@ in
       (mkRenamedOptionModule [ "boot" "systemd" "targets" ] [ "systemd" "targets" ])
       (mkRenamedOptionModule [ "boot" "systemd" "services" ] [ "systemd" "services" ])
     ];
-
 }
diff --git a/nixos/modules/system/boot/timesyncd.nix b/nixos/modules/system/boot/timesyncd.nix
index 8d8bfe5900a..8282cdd6f3a 100644
--- a/nixos/modules/system/boot/timesyncd.nix
+++ b/nixos/modules/system/boot/timesyncd.nix
@@ -40,6 +40,15 @@ with lib;
     users.users.systemd-timesync.uid = config.ids.uids.systemd-timesync;
     users.groups.systemd-timesync.gid = config.ids.gids.systemd-timesync;
 
+    system.activationScripts.systemd-timesyncd-migration = mkIf (versionOlder config.system.stateVersion "19.09") ''
+      # workaround an issue of systemd-timesyncd not starting due to upstream systemd reverting their dynamic users changes
+      #  - https://github.com/NixOS/nixpkgs/pull/61321#issuecomment-492423742
+      #  - https://github.com/systemd/systemd/issues/12131
+      if [ -L /var/lib/systemd/timesync ]; then
+        rm /var/lib/systemd/timesync
+        mv /var/lib/private/systemd/timesync /var/lib/systemd/timesync
+      fi
+    '';
   };
 
 }
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 9bce49c9e30..ddc253adbd0 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -237,6 +237,7 @@ in
   syncthing-relay = handleTest ./syncthing-relay.nix {};
   systemd = handleTest ./systemd.nix {};
   systemd-confinement = handleTest ./systemd-confinement.nix {};
+  systemd-timesyncd = handleTest ./systemd-timesyncd.nix {};
   pdns-recursor = handleTest ./pdns-recursor.nix {};
   taskserver = handleTest ./taskserver.nix {};
   telegraf = handleTest ./telegraf.nix {};
diff --git a/nixos/tests/systemd-timesyncd.nix b/nixos/tests/systemd-timesyncd.nix
new file mode 100644
index 00000000000..d12b8eb2bf7
--- /dev/null
+++ b/nixos/tests/systemd-timesyncd.nix
@@ -0,0 +1,52 @@
+# Regression test for systemd-timesync having moved the state directory without
+# upstream providing a migration path. https://github.com/systemd/systemd/issues/12131
+
+import ./make-test.nix (let
+  common = { lib, ... }: {
+    # override the `false` value from the qemu-vm base profile
+    services.timesyncd.enable = lib.mkForce true;
+  };
+  mkVM = conf: { imports = [ conf common ]; };
+in {
+  name = "systemd-timesyncd";
+  nodes = {
+    current = mkVM {};
+    pre1909 = mkVM ({lib, ... }: with lib; {
+      # create the path that should be migrated by our activation script when
+      # upgrading to a newer nixos version
+      system.stateVersion = "19.03";
+      system.activationScripts.simulate-old-timesync-state-dir = mkBefore ''
+        rm -f /var/lib/systemd/timesync
+        mkdir -p /var/lib/systemd /var/lib/private/systemd/timesync
+        ln -s /var/lib/private/systemd/timesync /var/lib/systemd/timesync
+        chown systemd-timesync: /var/lib/private/systemd/timesync
+      '';
+    });
+  };
+
+  testScript = ''
+    startAll;
+    $current->succeed('systemctl status systemd-timesyncd.service');
+    # on a new install with a recent systemd there should not be any
+    # leftovers from the dynamic user mess
+    $current->succeed('test -e /var/lib/systemd/timesync');
+    $current->succeed('test ! -L /var/lib/systemd/timesync');
+
+    # timesyncd should be running on the upgrading system since we fixed the
+    # file bits in the activation script
+    $pre1909->succeed('systemctl status systemd-timesyncd.service');
+
+    # the path should be gone after the migration
+    $pre1909->succeed('test ! -e /var/lib/private/systemd/timesync');
+
+    # and the new path should no longer be a symlink
+    $pre1909->succeed('test -e /var/lib/systemd/timesync');
+    $pre1909->succeed('test ! -L /var/lib/systemd/timesync');
+
+    # after a restart things should still work and not fail in the activation
+    # scripts and cause the boot to fail..
+    $pre1909->shutdown;
+    $pre1909->start;
+    $pre1909->succeed('systemctl status systemd-timesyncd.service');
+  '';
+})