Merge pull request #120489 from samueldr/fix/make-disk-image-auto-size

Fix make disk image automatic size

10 files changed, 91 insertions, 22 deletions

diff --git a/nixos/lib/make-disk-image.nix b/nixos/lib/make-disk-image.nix
index 88c87bcd568..7d40d3b5548 100644
--- a/nixos/lib/make-disk-image.nix
+++ b/nixos/lib/make-disk-image.nix
@@ -15,6 +15,8 @@
 
 , # size of the boot partition, is only used if partitionTableType is
   # either "efi" or "hybrid"
+  # This will be undersized slightly, as this is actually the offset of
+  # the end of the partition. Generally it will be 1MiB smaller.
   bootSize ? "256M"
 
 , # The files and directories to be placed in the target file system.
@@ -163,6 +165,8 @@ let format' = format; in let
 
   closureInfo = pkgs.closureInfo { rootPaths = [ config.system.build.toplevel channelSources ]; };
 
+  blockSize = toString (4 * 1024); # ext4fs block size (not block device sector size)
+
   prepareImage = ''
     export PATH=${binPath}
 
@@ -175,6 +179,24 @@ let format' = format; in let
       echo $(( "$1" * 512  ))
     }
 
+    # Given lines of numbers, adds them together
+    sum_lines() {
+      local acc=0
+      while read -r number; do
+        acc=$((acc+number))
+      done
+      echo "$acc"
+    }
+
+    mebibyte=$(( 1024 * 1024 ))
+
+    # Approximative percentage of reserved space in an ext4 fs over 512MiB.
+    # 0.05208587646484375
+    #  × 1000, integer part: 52
+    compute_fudge() {
+      echo $(( $1 * 52 / 1000 ))
+    }
+
     mkdir $out
 
     root="$PWD/root"
@@ -235,12 +257,53 @@ let format' = format; in let
 
     ${if diskSize == "auto" then ''
       ${if partitionTableType == "efi" || partitionTableType == "hybrid" then ''
-        additionalSpace=$(( ($(numfmt --from=iec '${additionalSpace}') + $(numfmt --from=iec '${bootSize}')) / 1000 ))
+        # Add the GPT at the end
+        gptSpace=$(( 512 * 34 * 1 ))
+        # Normally we'd need to account for alignment and things, if bootSize
+        # represented the actual size of the boot partition. But it instead
+        # represents the offset at which it ends.
+        # So we know bootSize is the reserved space in front of the partition.
+        reservedSpace=$(( gptSpace + $(numfmt --from=iec '${bootSize}') ))
+      '' else if partitionTableType == "legacy+gpt" then ''
+        # Add the GPT at the end
+        gptSpace=$(( 512 * 34 * 1 ))
+        # And include the bios_grub partition; the ext4 partition starts at 2MB exactly.
+        reservedSpace=$(( gptSpace + 2 * mebibyte ))
+      '' else if partitionTableType == "legacy" then ''
+        # Add the 1MiB aligned reserved space (includes MBR)
+        reservedSpace=$(( mebibyte ))
       '' else ''
-        additionalSpace=$(( $(numfmt --from=iec '${additionalSpace}') / 1000 ))
+        reservedSpace=0
       ''}
-      diskSize=$(( $(set -- $(du -d0 $root); echo "$1") + $additionalSpace ))
-      truncate -s "$diskSize"K $diskImage
+      additionalSpace=$(( $(numfmt --from=iec '${additionalSpace}') + reservedSpace ))
+
+      # Compute required space in filesystem blocks
+      diskUsage=$(find . ! -type d -exec 'du' '--apparent-size' '--block-size' "${blockSize}" '{}' ';' | cut -f1 | sum_lines)
+      # Each inode takes space!
+      numInodes=$(find . | wc -l)
+      # Convert to bytes, inodes take two blocks each!
+      diskUsage=$(( (diskUsage + 2 * numInodes) * ${blockSize} ))
+      # Then increase the required space to account for the reserved blocks.
+      fudge=$(compute_fudge $diskUsage)
+      requiredFilesystemSpace=$(( diskUsage + fudge ))
+
+      diskSize=$(( requiredFilesystemSpace  + additionalSpace ))
+
+      # Round up to the nearest mebibyte.
+      # This ensures whole 512 bytes sector sizes in the disk image
+      # and helps towards aligning partitions optimally.
+      if (( diskSize % mebibyte )); then
+        diskSize=$(( ( diskSize / mebibyte + 1) * mebibyte ))
+      fi
+
+      truncate -s "$diskSize" $diskImage
+
+      printf "Automatic disk size...\n"
+      printf "  Closure space use: %d bytes\n" $diskUsage
+      printf "  fudge: %d bytes\n" $fudge
+      printf "  Filesystem size needed: %d bytes\n" $requiredFilesystemSpace
+      printf "  Additional space: %d bytes\n" $additionalSpace
+      printf "  Disk image size: %d bytes\n" $diskSize
     '' else ''
       truncate -s ${toString diskSize}M $diskImage
     ''}
@@ -251,9 +314,9 @@ let format' = format; in let
       # Get start & length of the root partition in sectors to $START and $SECTORS.
       eval $(partx $diskImage -o START,SECTORS --nr ${rootPartition} --pairs)
 
-      mkfs.${fsType} -F -L ${label} $diskImage -E offset=$(sectorsToBytes $START) $(sectorsToKilobytes $SECTORS)K
+      mkfs.${fsType} -b ${blockSize} -F -L ${label} $diskImage -E offset=$(sectorsToBytes $START) $(sectorsToKilobytes $SECTORS)K
     '' else ''
-      mkfs.${fsType} -F -L ${label} $diskImage
+      mkfs.${fsType} -b ${blockSize} -F -L ${label} $diskImage
     ''}
 
     echo "copying staging root to image..."
diff --git a/nixos/maintainers/scripts/cloudstack/cloudstack-image.nix b/nixos/maintainers/scripts/cloudstack/cloudstack-image.nix
index 37b46db059c..005f75476e9 100644
--- a/nixos/maintainers/scripts/cloudstack/cloudstack-image.nix
+++ b/nixos/maintainers/scripts/cloudstack/cloudstack-image.nix
@@ -10,7 +10,6 @@ with lib;
 
   system.build.cloudstackImage = import ../../../lib/make-disk-image.nix {
     inherit lib config pkgs;
-    diskSize = 8192;
     format = "qcow2";
     configFile = pkgs.writeText "configuration.nix"
       ''
diff --git a/nixos/maintainers/scripts/ec2/amazon-image.nix b/nixos/maintainers/scripts/ec2/amazon-image.nix
index 0ecf07669a1..653744986d1 100644
--- a/nixos/maintainers/scripts/ec2/amazon-image.nix
+++ b/nixos/maintainers/scripts/ec2/amazon-image.nix
@@ -40,8 +40,9 @@ in {
     };
 
     sizeMB = mkOption {
-      type = types.int;
-      default = if config.ec2.hvm then 2048 else 8192;
+      type = with types; either (enum [ "auto" ]) int;
+      default = "auto";
+      example = 8192;
       description = "The size in MB of the image";
     };
 
diff --git a/nixos/maintainers/scripts/openstack/openstack-image.nix b/nixos/maintainers/scripts/openstack/openstack-image.nix
index 4c464f43f61..3255e7f3d44 100644
--- a/nixos/maintainers/scripts/openstack/openstack-image.nix
+++ b/nixos/maintainers/scripts/openstack/openstack-image.nix
@@ -12,8 +12,8 @@ with lib;
 
   system.build.openstackImage = import ../../../lib/make-disk-image.nix {
     inherit lib config;
+    additionalSpace = "1024M";
     pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
-    diskSize = 8192;
     format = "qcow2";
     configFile = pkgs.writeText "configuration.nix"
       ''
diff --git a/nixos/modules/virtualisation/azure-image.nix b/nixos/modules/virtualisation/azure-image.nix
index 60fed3222ef..03dd3c05130 100644
--- a/nixos/modules/virtualisation/azure-image.nix
+++ b/nixos/modules/virtualisation/azure-image.nix
@@ -9,8 +9,9 @@ in
 
   options = {
     virtualisation.azureImage.diskSize = mkOption {
-      type = with types; int;
-      default = 2048;
+      type = with types; either (enum [ "auto" ]) int;
+      default = "auto";
+      example = 2048;
       description = ''
         Size of disk image. Unit is MB.
       '';
diff --git a/nixos/modules/virtualisation/digital-ocean-image.nix b/nixos/modules/virtualisation/digital-ocean-image.nix
index b582e235d43..0ff2ee591f2 100644
--- a/nixos/modules/virtualisation/digital-ocean-image.nix
+++ b/nixos/modules/virtualisation/digital-ocean-image.nix
@@ -10,8 +10,9 @@ in
 
   options = {
     virtualisation.digitalOceanImage.diskSize = mkOption {
-      type = with types; int;
-      default = 4096;
+      type = with types; either (enum [ "auto" ]) int;
+      default = "auto";
+      example = 4096;
       description = ''
         Size of disk image. Unit is MB.
       '';
diff --git a/nixos/modules/virtualisation/google-compute-image.nix b/nixos/modules/virtualisation/google-compute-image.nix
index e2332df611a..79c3921669e 100644
--- a/nixos/modules/virtualisation/google-compute-image.nix
+++ b/nixos/modules/virtualisation/google-compute-image.nix
@@ -18,8 +18,9 @@ in
 
   options = {
     virtualisation.googleComputeImage.diskSize = mkOption {
-      type = with types; int;
-      default = 1536;
+      type = with types; either (enum [ "auto" ]) int;
+      default = "auto";
+      example = 1536;
       description = ''
         Size of disk image. Unit is MB.
       '';
diff --git a/nixos/modules/virtualisation/hyperv-image.nix b/nixos/modules/virtualisation/hyperv-image.nix
index fabc9113dfc..6845d675009 100644
--- a/nixos/modules/virtualisation/hyperv-image.nix
+++ b/nixos/modules/virtualisation/hyperv-image.nix
@@ -9,8 +9,9 @@ in {
   options = {
     hyperv = {
       baseImageSize = mkOption {
-        type = types.int;
-        default = 2048;
+        type = with types; either (enum [ "auto" ]) int;
+        default = "auto";
+        example = 2048;
         description = ''
           The size of the hyper-v base image in MiB.
         '';
diff --git a/nixos/modules/virtualisation/virtualbox-image.nix b/nixos/modules/virtualisation/virtualbox-image.nix
index fa580e8b42d..071edda8269 100644
--- a/nixos/modules/virtualisation/virtualbox-image.nix
+++ b/nixos/modules/virtualisation/virtualbox-image.nix
@@ -11,8 +11,9 @@ in {
   options = {
     virtualbox = {
       baseImageSize = mkOption {
-        type = types.int;
-        default = 50 * 1024;
+        type = with types; either (enum [ "auto" ]) int;
+        default = "auto";
+        example = 50 * 1024;
         description = ''
           The size of the VirtualBox base image in MiB.
         '';
diff --git a/nixos/modules/virtualisation/vmware-image.nix b/nixos/modules/virtualisation/vmware-image.nix
index 9da9e145f7a..f6cd12e2bb7 100644
--- a/nixos/modules/virtualisation/vmware-image.nix
+++ b/nixos/modules/virtualisation/vmware-image.nix
@@ -18,8 +18,9 @@ in {
   options = {
     vmware = {
       baseImageSize = mkOption {
-        type = types.int;
-        default = 2048;
+        type = with types; either (enum [ "auto" ]) int;
+        default = "auto";
+        example = 2048;
         description = ''
           The size of the VMWare base image in MiB.
         '';