diff options
author | Sergei Trofimovich <slyich@gmail.com> | 2021-09-24 08:06:57 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-09-24 01:06:57 -0700 |
commit | 39e8ec2db68b863543bd377e44fbe02f8d05864e (patch) | |
tree | 8ff2d5ae709186d0b7c9faed00d5d65095dbaa85 /maintainers/scripts/rebuild-amount.sh | |
parent | 1922e8792421a31f4d4ca962ee2d35cd1b72bd18 (diff) | |
download | nixpkgs-39e8ec2db68b863543bd377e44fbe02f8d05864e.tar nixpkgs-39e8ec2db68b863543bd377e44fbe02f8d05864e.tar.gz nixpkgs-39e8ec2db68b863543bd377e44fbe02f8d05864e.tar.bz2 nixpkgs-39e8ec2db68b863543bd377e44fbe02f8d05864e.tar.lz nixpkgs-39e8ec2db68b863543bd377e44fbe02f8d05864e.tar.xz nixpkgs-39e8ec2db68b863543bd377e44fbe02f8d05864e.tar.zst nixpkgs-39e8ec2db68b863543bd377e44fbe02f8d05864e.zip |
maintainers/scripts/rebuild-amount.sh: report parallelism, add example, cleanup (#137695)
* maintainers/scripts/rebuild-amount.sh: report parallelism setting This slightly helps discovering parallelism support. * maintainers/scripts/rebuild-amount.sh: add basic example It's not clear if script should be used against HEAD or HEAD^. Let's have a copyable example. * maintainers/scripts/rebuild-amount.sh: convert tabs to four spaces One notable change is here-document conversion. Before: <TAB>cat <<-FOO <TAB>... <TAB>FOO After: <sp><sp><sp><sp>cat <<FOO <sp><sp><sp><sp>... FOO Note seemingly misaligned 'FOO': '-FOO' understands leading tabs, but not spaces.
Diffstat (limited to 'maintainers/scripts/rebuild-amount.sh')
-rwxr-xr-x | maintainers/scripts/rebuild-amount.sh | 165 |
1 files changed, 86 insertions, 79 deletions
diff --git a/maintainers/scripts/rebuild-amount.sh b/maintainers/scripts/rebuild-amount.sh index ca89e08073e..bedd352db5f 100755 --- a/maintainers/scripts/rebuild-amount.sh +++ b/maintainers/scripts/rebuild-amount.sh @@ -4,123 +4,130 @@ set -e # --print: avoid dependency on environment optPrint= if [ "$1" == "--print" ]; then - optPrint=true - shift + optPrint=true + shift fi if [ "$#" != 1 ] && [ "$#" != 2 ]; then - cat <<-EOF - Usage: $0 [--print] commit-spec [commit-spec] - You need to be in a git-controlled nixpkgs tree. - The current state of the tree will be used if the second commit is missing. - EOF - exit 1 + cat <<EOF + Usage: $0 [--print] from-commit-spec [to-commit-spec] + You need to be in a git-controlled nixpkgs tree. + The current state of the tree will be used if the second commit is missing. + + Examples: + effect of latest commit: + $ $0 HEAD^ + $ $0 --print HEAD^ + effect of the whole patch series for 'staging' branch: + $ $0 origin/staging staging +EOF + exit 1 fi # A slightly hacky way to get the config. parallel="$(echo 'config.rebuild-amount.parallel or false' | nix-repl . 2>/dev/null \ - | grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)" + | grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)" -echo "Estimating rebuild amount by counting changed Hydra jobs." +echo "Estimating rebuild amount by counting changed Hydra jobs (parallel=${parallel:-unset})." toRemove=() cleanup() { - rm -rf "${toRemove[@]}" + rm -rf "${toRemove[@]}" } trap cleanup EXIT SIGINT SIGQUIT ERR MKTEMP='mktemp --tmpdir nix-rebuild-amount-XXXXXXXX' nixexpr() { - cat <<-EONIX - let - lib = import $1/lib; - hydraJobs = import $1/pkgs/top-level/release.nix - # Compromise: accuracy vs. resources needed for evaluation. - { supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; }; - cfg = (import $1 {}).config.rebuild-amount or {}; - - recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; }; - - # hydraJobs leaves recurseForDerivations as empty attrmaps; - # that would break nix-env and we also need to recurse everywhere. - tweak = lib.mapAttrs - (name: val: - if name == "recurseForDerivations" then true - else if lib.isAttrs val && val.type or null != "derivation" - then recurseIntoAttrs (tweak val) - else val - ); - - # Some of these contain explicit references to platform(s) we want to avoid; - # some even (transitively) depend on ~/.nixpkgs/config.nix (!) - blacklist = [ - "tarball" "metrics" "manual" - "darwin-tested" "unstable" "stdenvBootstrapTools" - "moduleSystem" "lib-tests" # these just confuse the output - ]; - - in - tweak (builtins.removeAttrs hydraJobs blacklist) - EONIX + cat <<EONIX + let + lib = import $1/lib; + hydraJobs = import $1/pkgs/top-level/release.nix + # Compromise: accuracy vs. resources needed for evaluation. + { supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; }; + cfg = (import $1 {}).config.rebuild-amount or {}; + + recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; }; + + # hydraJobs leaves recurseForDerivations as empty attrmaps; + # that would break nix-env and we also need to recurse everywhere. + tweak = lib.mapAttrs + (name: val: + if name == "recurseForDerivations" then true + else if lib.isAttrs val && val.type or null != "derivation" + then recurseIntoAttrs (tweak val) + else val + ); + + # Some of these contain explicit references to platform(s) we want to avoid; + # some even (transitively) depend on ~/.nixpkgs/config.nix (!) + blacklist = [ + "tarball" "metrics" "manual" + "darwin-tested" "unstable" "stdenvBootstrapTools" + "moduleSystem" "lib-tests" # these just confuse the output + ]; + + in + tweak (builtins.removeAttrs hydraJobs blacklist) +EONIX } # Output packages in tree $2 that weren't in $1. # Changing the output hash or name is taken as a change. # Extra nix-env parameters can be in $3 newPkgs() { - # We use files instead of pipes, as running multiple nix-env processes - # could eat too much memory for a standard 4GiB machine. - local -a list - for i in 1 2; do - local l="$($MKTEMP)" - list[$i]="$l" - toRemove+=("$l") - - local expr="$($MKTEMP)" - toRemove+=("$expr") - nixexpr "${!i}" > "$expr" - - nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \ - | sort > "${list[$i]}" & - - if [ "$parallel" != "true" ]; then - wait - fi - done - - wait - comm -13 "${list[@]}" + # We use files instead of pipes, as running multiple nix-env processes + # could eat too much memory for a standard 4GiB machine. + local -a list + for i in 1 2; do + local l="$($MKTEMP)" + list[$i]="$l" + toRemove+=("$l") + + local expr="$($MKTEMP)" + toRemove+=("$expr") + nixexpr "${!i}" > "$expr" + + nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \ + | sort > "${list[$i]}" & + + if [ "$parallel" != "true" ]; then + wait + fi + done + + wait + comm -13 "${list[@]}" } # Prepare nixpkgs trees. declare -a tree for i in 1 2; do - if [ -n "${!i}" ]; then # use the given commit - dir="$($MKTEMP -d)" - tree[$i]="$dir" - toRemove+=("$dir") - - git clone --shared --no-checkout --quiet . "${tree[$i]}" - (cd "${tree[$i]}" && git checkout --quiet "${!i}") - else #use the current tree - tree[$i]="$(pwd)" - fi + if [ -n "${!i}" ]; then # use the given commit + dir="$($MKTEMP -d)" + tree[$i]="$dir" + toRemove+=("$dir") + + git clone --shared --no-checkout --quiet . "${tree[$i]}" + (cd "${tree[$i]}" && git checkout --quiet "${!i}") + else #use the current tree + tree[$i]="$(pwd)" + fi done newlist="$($MKTEMP)" toRemove+=("$newlist") # Notes: -# - the evaluation is done on x86_64-linux, like on Hydra. -# - using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove) +# - the evaluation is done on x86_64-linux, like on Hydra. +# - using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove) newPkgs "${tree[1]}" "${tree[2]}" '--argstr system "x86_64-linux"' > "$newlist" # Hacky: keep only the last word of each attribute path and sort. sed -n 's/\([^. ]*\.\)*\([^. ]*\) .*$/\2/p' < "$newlist" \ - | sort | uniq -c + | sort | uniq -c if [ -n "$optPrint" ]; then - echo - cat "$newlist" + echo + cat "$newlist" fi |