diff options
Diffstat (limited to 'maintainers/scripts/haskell')
9 files changed, 662 insertions, 0 deletions
diff --git a/maintainers/scripts/haskell/hydra-report.hs b/maintainers/scripts/haskell/hydra-report.hs new file mode 100755 index 00000000000..fd6430d43c9 --- /dev/null +++ b/maintainers/scripts/haskell/hydra-report.hs @@ -0,0 +1,408 @@ +#! /usr/bin/env nix-shell +#! nix-shell -p "haskellPackages.ghcWithPackages (p: [p.aeson p.req])" +#! nix-shell -p hydra-unstable +#! nix-shell -i runhaskell + +{- + +The purpose of this script is + +1) download the state of the nixpkgs/haskell-updates job from hydra (with get-report) +2) print a summary of the state suitable for pasting into a github comment (with ping-maintainers) +3) print a list of broken packages suitable for pasting into configuration-hackage2nix.yaml + +Because step 1) is quite expensive and takes roughly ~5 minutes the result is cached in a json file in XDG_CACHE. + +-} +{-# LANGUAGE BlockArguments #-} +{-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE DerivingStrategies #-} +{-# LANGUAGE DuplicateRecordFields #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE MultiWayIf #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TupleSections #-} +{-# OPTIONS_GHC -Wall #-} + +import Control.Monad (forM_, (<=<)) +import Control.Monad.Trans (MonadIO (liftIO)) +import Data.Aeson ( + FromJSON, + ToJSON, + decodeFileStrict', + eitherDecodeStrict', + encodeFile, + ) +import Data.Foldable (Foldable (toList), foldl') +import Data.List.NonEmpty (NonEmpty, nonEmpty) +import qualified Data.List.NonEmpty as NonEmpty +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Maybe (fromMaybe, mapMaybe) +import Data.Monoid (Sum (Sum, getSum)) +import Data.Sequence (Seq) +import qualified Data.Sequence as Seq +import Data.Set (Set) +import qualified Data.Set as Set +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Text.Encoding (encodeUtf8) +import Data.Time (defaultTimeLocale, formatTime, getCurrentTime) +import Data.Time.Clock (UTCTime) +import GHC.Generics (Generic) +import Network.HTTP.Req ( + GET (GET), + NoReqBody (NoReqBody), + defaultHttpConfig, + header, + https, + jsonResponse, + req, + responseBody, + responseTimeout, + runReq, + (/:), + ) +import System.Directory (XdgDirectory (XdgCache), getXdgDirectory) +import System.Environment (getArgs) +import System.Process (readProcess) +import Prelude hiding (id) + +newtype JobsetEvals = JobsetEvals + { evals :: Seq Eval + } + deriving (Generic, ToJSON, FromJSON, Show) + +newtype Nixpkgs = Nixpkgs {revision :: Text} + deriving (Generic, ToJSON, FromJSON, Show) + +newtype JobsetEvalInputs = JobsetEvalInputs {nixpkgs :: Nixpkgs} + deriving (Generic, ToJSON, FromJSON, Show) + +data Eval = Eval + { id :: Int + , jobsetevalinputs :: JobsetEvalInputs + } + deriving (Generic, ToJSON, FromJSON, Show) + +data Build = Build + { job :: Text + , buildstatus :: Maybe Int + , finished :: Int + , id :: Int + , nixname :: Text + , system :: Text + , jobsetevals :: Seq Int + } + deriving (Generic, ToJSON, FromJSON, Show) + +main :: IO () +main = do + args <- getArgs + case args of + ["get-report"] -> getBuildReports + ["ping-maintainers"] -> printMaintainerPing + ["mark-broken-list"] -> printMarkBrokenList + _ -> putStrLn "Usage: get-report | ping-maintainers | mark-broken-list" + +reportFileName :: IO FilePath +reportFileName = getXdgDirectory XdgCache "haskell-updates-build-report.json" + +showT :: Show a => a -> Text +showT = Text.pack . show + +getBuildReports :: IO () +getBuildReports = runReq defaultHttpConfig do + evalMay <- Seq.lookup 0 . evals <$> myReq (https "hydra.nixos.org" /: "jobset" /: "nixpkgs" /: "haskell-updates" /: "evals") mempty + eval@Eval{id} <- maybe (liftIO $ fail "No Evalution found") pure evalMay + liftIO . putStrLn $ "Fetching evaluation " <> show id <> " from Hydra. This might take a few minutes..." + buildReports :: Seq Build <- myReq (https "hydra.nixos.org" /: "eval" /: showT id /: "builds") (responseTimeout 600000000) + liftIO do + fileName <- reportFileName + putStrLn $ "Finished fetching all builds from Hydra, saving report as " <> fileName + now <- getCurrentTime + encodeFile fileName (eval, now, buildReports) + where + myReq query option = responseBody <$> req GET query NoReqBody jsonResponse (header "User-Agent" "hydra-report.hs/v1 (nixpkgs;maintainers/scripts/haskell)" <> option) + +hydraEvalCommand :: FilePath +hydraEvalCommand = "hydra-eval-jobs" + +hydraEvalParams :: [String] +hydraEvalParams = ["-I", ".", "pkgs/top-level/release-haskell.nix"] + +handlesCommand :: FilePath +handlesCommand = "nix-instantiate" + +handlesParams :: [String] +handlesParams = ["--eval", "--strict", "--json", "-"] + +handlesExpression :: String +handlesExpression = "with import ./. {}; with lib; zipAttrsWith (_: builtins.head) (mapAttrsToList (_: v: if v ? github then { \"${v.email}\" = v.github; } else {}) (import maintainers/maintainer-list.nix))" + +-- | This newtype is used to parse a Hydra job output from @hydra-eval-jobs@. +-- The only field we are interested in is @maintainers@, which is why this +-- is just a newtype. +-- +-- Note that there are occassionally jobs that don't have a maintainers +-- field, which is why this has to be @Maybe Text@. +newtype Maintainers = Maintainers { maintainers :: Maybe Text } + deriving stock (Generic, Show) + deriving anyclass (FromJSON, ToJSON) + +-- | This is a 'Map' from Hydra job name to maintainer email addresses. +-- +-- It has values similar to the following: +-- +-- @@ +-- fromList +-- [ ("arion.aarch64-linux", Maintainers (Just "robert@example.com")) +-- , ("bench.x86_64-linux", Maintainers (Just "")) +-- , ("conduit.x86_64-linux", Maintainers (Just "snoy@man.com, web@ber.com")) +-- , ("lens.x86_64-darwin", Maintainers (Just "ek@category.com")) +-- ] +-- @@ +-- +-- Note that Hydra jobs without maintainers will have an empty string for the +-- maintainer list. +type HydraJobs = Map Text Maintainers + +-- | Map of email addresses to GitHub handles. +-- This is built from the file @../../maintainer-list.nix@. +-- +-- It has values similar to the following: +-- +-- @@ +-- fromList +-- [ ("robert@example.com", "rob22") +-- , ("ek@category.com", "edkm") +-- ] +-- @@ +type EmailToGitHubHandles = Map Text Text + +-- | Map of Hydra jobs to maintainer GitHub handles. +-- +-- It has values similar to the following: +-- +-- @@ +-- fromList +-- [ ("arion.aarch64-linux", ["rob22"]) +-- , ("conduit.x86_64-darwin", ["snoyb", "webber"]) +-- ] +-- @@ +type MaintainerMap = Map Text (NonEmpty Text) + +-- | Generate a mapping of Hydra job names to maintainer GitHub handles. +getMaintainerMap :: IO MaintainerMap +getMaintainerMap = do + hydraJobs :: HydraJobs <- + readJSONProcess hydraEvalCommand hydraEvalParams "" "Failed to decode hydra-eval-jobs output: " + handlesMap :: EmailToGitHubHandles <- + readJSONProcess handlesCommand handlesParams handlesExpression "Failed to decode nix output for lookup of github handles: " + pure $ Map.mapMaybe (splitMaintainersToGitHubHandles handlesMap) hydraJobs + where + -- Split a comma-spearated string of Maintainers into a NonEmpty list of + -- GitHub handles. + splitMaintainersToGitHubHandles + :: EmailToGitHubHandles -> Maintainers -> Maybe (NonEmpty Text) + splitMaintainersToGitHubHandles handlesMap (Maintainers maint) = + nonEmpty . mapMaybe (`Map.lookup` handlesMap) . Text.splitOn ", " $ fromMaybe "" maint + +-- | Run a process that produces JSON on stdout and and decode the JSON to a +-- data type. +-- +-- If the JSON-decoding fails, throw the JSON-decoding error. +readJSONProcess + :: FromJSON a + => FilePath -- ^ Filename of executable. + -> [String] -- ^ Arguments + -> String -- ^ stdin to pass to the process + -> String -- ^ String to prefix to JSON-decode error. + -> IO a +readJSONProcess exe args input err = do + output <- readProcess exe args input + let eitherDecodedOutput = eitherDecodeStrict' . encodeUtf8 . Text.pack $ output + case eitherDecodedOutput of + Left decodeErr -> error $ err <> decodeErr <> "\nRaw: '" <> take 1000 output <> "'" + Right decodedOutput -> pure decodedOutput + +-- BuildStates are sorted by subjective importance/concerningness +data BuildState + = Failed + | DependencyFailed + | OutputLimitExceeded + | Unknown (Maybe Int) + | TimedOut + | Canceled + | HydraFailure + | Unfinished + | Success + deriving stock (Show, Eq, Ord) + +icon :: BuildState -> Text +icon = \case + Failed -> ":x:" + DependencyFailed -> ":heavy_exclamation_mark:" + OutputLimitExceeded -> ":warning:" + Unknown x -> "unknown code " <> showT x + TimedOut -> ":hourglass::no_entry_sign:" + Canceled -> ":no_entry_sign:" + Unfinished -> ":hourglass_flowing_sand:" + HydraFailure -> ":construction:" + Success -> ":heavy_check_mark:" + +platformIcon :: Platform -> Text +platformIcon (Platform x) = case x of + "x86_64-linux" -> ":penguin:" + "aarch64-linux" -> ":iphone:" + "x86_64-darwin" -> ":apple:" + _ -> x + +data BuildResult = BuildResult {state :: BuildState, id :: Int} deriving (Show, Eq, Ord) +newtype Platform = Platform {platform :: Text} deriving (Show, Eq, Ord) +newtype Table row col a = Table (Map (row, col) a) +type StatusSummary = Map Text (Table Text Platform BuildResult, Set Text) + +instance (Ord row, Ord col, Semigroup a) => Semigroup (Table row col a) where + Table l <> Table r = Table (Map.unionWith (<>) l r) +instance (Ord row, Ord col, Semigroup a) => Monoid (Table row col a) where + mempty = Table Map.empty +instance Functor (Table row col) where + fmap f (Table a) = Table (fmap f a) +instance Foldable (Table row col) where + foldMap f (Table a) = foldMap f a + +buildSummary :: MaintainerMap -> Seq Build -> StatusSummary +buildSummary maintainerMap = foldl (Map.unionWith unionSummary) Map.empty . fmap toSummary + where + unionSummary (Table l, l') (Table r, r') = (Table $ Map.union l r, l' <> r') + toSummary Build{finished, buildstatus, job, id, system} = Map.singleton name (Table (Map.singleton (set, Platform system) (BuildResult state id)), maintainers) + where + state :: BuildState + state = case (finished, buildstatus) of + (0, _) -> Unfinished + (_, Just 0) -> Success + (_, Just 1) -> Failed + (_, Just 2) -> DependencyFailed + (_, Just 3) -> HydraFailure + (_, Just 4) -> Canceled + (_, Just 7) -> TimedOut + (_, Just 11) -> OutputLimitExceeded + (_, i) -> Unknown i + packageName = fromMaybe job (Text.stripSuffix ("." <> system) job) + splitted = nonEmpty $ Text.splitOn "." packageName + name = maybe packageName NonEmpty.last splitted + set = maybe "" (Text.intercalate "." . NonEmpty.init) splitted + maintainers = maybe mempty (Set.fromList . toList) (Map.lookup job maintainerMap) + +readBuildReports :: IO (Eval, UTCTime, Seq Build) +readBuildReports = do + file <- reportFileName + fromMaybe (error $ "Could not decode " <> file) <$> decodeFileStrict' file + +sep :: Text +sep = " | " +joinTable :: [Text] -> Text +joinTable t = sep <> Text.intercalate sep t <> sep + +type NumSummary = Table Platform BuildState Int + +printTable :: (Ord rows, Ord cols) => Text -> (rows -> Text) -> (cols -> Text) -> (entries -> Text) -> Table rows cols entries -> [Text] +printTable name showR showC showE (Table mapping) = joinTable <$> (name : map showC cols) : replicate (length cols + sepsInName + 1) "---" : map printRow rows + where + sepsInName = Text.count "|" name + printRow row = showR row : map (\col -> maybe "" showE (Map.lookup (row, col) mapping)) cols + rows = toList $ Set.fromList (fst <$> Map.keys mapping) + cols = toList $ Set.fromList (snd <$> Map.keys mapping) + +printJob :: Int -> Text -> (Table Text Platform BuildResult, Text) -> [Text] +printJob evalId name (Table mapping, maintainers) = + if length sets <= 1 + then map printSingleRow sets + else ["- [ ] " <> makeJobSearchLink "" name <> " " <> maintainers] <> map printRow sets + where + printRow set = " - " <> printState set <> " " <> makeJobSearchLink set (if Text.null set then "toplevel" else set) + printSingleRow set = "- [ ] " <> printState set <> " " <> makeJobSearchLink set (makePkgName set) <> " " <> maintainers + makePkgName set = (if Text.null set then "" else set <> ".") <> name + printState set = Text.intercalate " " $ map (\pf -> maybe "" (label pf) $ Map.lookup (set, pf) mapping) platforms + makeJobSearchLink set linkLabel= makeSearchLink evalId linkLabel (makePkgName set) + sets = toList $ Set.fromList (fst <$> Map.keys mapping) + platforms = toList $ Set.fromList (snd <$> Map.keys mapping) + label pf (BuildResult s i) = "[[" <> platformIcon pf <> icon s <> "]](https://hydra.nixos.org/build/" <> showT i <> ")" + +makeSearchLink :: Int -> Text -> Text -> Text +makeSearchLink evalId linkLabel query = "[" <> linkLabel <> "](" <> "https://hydra.nixos.org/eval/" <> showT evalId <> "?filter=" <> query <> ")" + +statusToNumSummary :: StatusSummary -> NumSummary +statusToNumSummary = fmap getSum . foldMap (fmap Sum . jobTotals) + +jobTotals :: (Table Text Platform BuildResult, a) -> Table Platform BuildState Int +jobTotals (Table mapping, _) = getSum <$> Table (Map.foldMapWithKey (\(_, platform) (BuildResult buildstate _) -> Map.singleton (platform, buildstate) (Sum 1)) mapping) + +details :: Text -> [Text] -> [Text] +details summary content = ["<details><summary>" <> summary <> " </summary>", ""] <> content <> ["</details>", ""] + +printBuildSummary :: Eval -> UTCTime -> StatusSummary -> Text +printBuildSummary + Eval{id, jobsetevalinputs = JobsetEvalInputs{nixpkgs = Nixpkgs{revision}}} + fetchTime + summary = + Text.unlines $ + headline <> totals + <> optionalList "#### Maintained packages with build failure" (maintainedList fails) + <> optionalList "#### Maintained packages with failed dependency" (maintainedList failedDeps) + <> optionalList "#### Maintained packages with unknown error" (maintainedList unknownErr) + <> optionalHideableList "#### Unmaintained packages with build failure" (unmaintainedList fails) + <> optionalHideableList "#### Unmaintained packages with failed dependency" (unmaintainedList failedDeps) + <> optionalHideableList "#### Unmaintained packages with unknown error" (unmaintainedList unknownErr) + <> footer + where + footer = ["*Report generated with [maintainers/scripts/haskell/hydra-report.hs](https://github.com/NixOS/nixpkgs/blob/haskell-updates/maintainers/scripts/haskell/hydra-report.sh)*"] + totals = + [ "#### Build summary" + , "" + ] + <> printTable "Platform" (\x -> makeSearchLink id (platform x <> " " <> platformIcon x) ("." <> platform x)) (\x -> showT x <> " " <> icon x) showT (statusToNumSummary summary) + headline = + [ "### [haskell-updates build report from hydra](https://hydra.nixos.org/jobset/nixpkgs/haskell-updates)" + , "*evaluation [" + <> showT id + <> "](https://hydra.nixos.org/eval/" + <> showT id + <> ") of nixpkgs commit [" + <> Text.take 7 revision + <> "](https://github.com/NixOS/nixpkgs/commits/" + <> revision + <> ") as of " + <> Text.pack (formatTime defaultTimeLocale "%Y-%m-%d %H:%M UTC" fetchTime) + <> "*" + ] + jobsByState predicate = Map.filter (predicate . foldl' min Success . fmap state . fst) summary + fails = jobsByState (== Failed) + failedDeps = jobsByState (== DependencyFailed) + unknownErr = jobsByState (\x -> x > DependencyFailed && x < TimedOut) + withMaintainer = Map.mapMaybe (\(x, m) -> (x,) <$> nonEmpty (Set.toList m)) + withoutMaintainer = Map.mapMaybe (\(x, m) -> if Set.null m then Just x else Nothing) + optionalList heading list = if null list then mempty else [heading] <> list + optionalHideableList heading list = if null list then mempty else [heading] <> details (showT (length list) <> " job(s)") list + maintainedList = showMaintainedBuild <=< Map.toList . withMaintainer + unmaintainedList = showBuild <=< Map.toList . withoutMaintainer + showBuild (name, table) = printJob id name (table, "") + showMaintainedBuild (name, (table, maintainers)) = printJob id name (table, Text.intercalate " " (fmap ("@" <>) (toList maintainers))) + +printMaintainerPing :: IO () +printMaintainerPing = do + maintainerMap <- getMaintainerMap + (eval, fetchTime, buildReport) <- readBuildReports + putStrLn (Text.unpack (printBuildSummary eval fetchTime (buildSummary maintainerMap buildReport))) + +printMarkBrokenList :: IO () +printMarkBrokenList = do + (_, _, buildReport) <- readBuildReports + forM_ buildReport \Build{buildstatus, job} -> + case (buildstatus, Text.splitOn "." job) of + (Just 1, ["haskellPackages", name, "x86_64-linux"]) -> putStrLn $ " - " <> Text.unpack name + _ -> pure () diff --git a/maintainers/scripts/haskell/mark-broken.sh b/maintainers/scripts/haskell/mark-broken.sh new file mode 100755 index 00000000000..97dd5be8aaa --- /dev/null +++ b/maintainers/scripts/haskell/mark-broken.sh @@ -0,0 +1,47 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p coreutils git -I nixpkgs=. + +# This script uses the data pulled with +# maintainers/scripts/haskell/hydra-report.hs get-report to produce a list of +# failing builds that get written to the hackage2nix config. Then +# hackage-packages.nix gets regenerated and transitive-broken packages get +# marked as dont-distribute in the config as well. +# This should disable builds for most failing jobs in the haskell-updates jobset. + +set -euo pipefail + +broken_config="pkgs/development/haskell-modules/configuration-hackage2nix/broken.yaml" + +tmpfile=$(mktemp) +trap "rm ${tmpfile}" 0 + +echo "Remember that you need to manually run 'maintainers/scripts/haskell/hydra-report.hs get-report' sometime before running this script." +echo "Generating a list of broken builds and displaying for manual confirmation ..." +maintainers/scripts/haskell/hydra-report.hs mark-broken-list | sort -i > "$tmpfile" + +$EDITOR "$tmpfile" + +tail -n +3 "$broken_config" >> "$tmpfile" + +cat > "$broken_config" << EOF +broken-packages: + # These packages don't compile. +EOF + +# clear environment here to avoid things like allowing broken builds in +sort -iu "$tmpfile" >> "$broken_config" +clear="env -u HOME -u NIXPKGS_CONFIG" +$clear maintainers/scripts/haskell/regenerate-hackage-packages.sh +$clear maintainers/scripts/haskell/regenerate-transitive-broken-packages.sh +$clear maintainers/scripts/haskell/regenerate-hackage-packages.sh + +if [[ "${1:-}" == "--do-commit" ]]; then +git add $broken_config +git add pkgs/development/haskell-modules/configuration-hackage2nix/transitive-broken.yaml +git add pkgs/development/haskell-modules/hackage-packages.nix +git commit -F - << EOF +haskellPackages: mark builds failing on hydra as broken + +This commit has been generated by maintainers/scripts/haskell/mark-broken.sh +EOF +fi diff --git a/maintainers/scripts/haskell/regenerate-hackage-packages.sh b/maintainers/scripts/haskell/regenerate-hackage-packages.sh new file mode 100755 index 00000000000..285f6ed7cf9 --- /dev/null +++ b/maintainers/scripts/haskell/regenerate-hackage-packages.sh @@ -0,0 +1,46 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p coreutils haskellPackages.cabal2nix-unstable git nix -I nixpkgs=. + +# This script is used to regenerate nixpkgs' Haskell package set, using the +# tool hackage2nix from the nixos/cabal2nix repo. hackage2nix looks at the +# config files in pkgs/development/haskell-modules/configuration-hackage2nix +# and generates a Nix expression for package version specified there, using the +# Cabal files from the Hackage database (available under all-cabal-hashes) and +# its companion tool cabal2nix. +# +# Related scripts are update-hackage.sh, for updating the snapshot of the +# Hackage database used by hackage2nix, and update-cabal2nix-unstable.sh, +# for updating the version of hackage2nix used to perform this task. + +set -euo pipefail + +HACKAGE2NIX="${HACKAGE2NIX:-hackage2nix}" + +# To prevent hackage2nix fails because of encoding. +# See: https://github.com/NixOS/nixpkgs/pull/122023 +export LC_ALL=C.UTF-8 + +extraction_derivation='with import ./. {}; runCommand "unpacked-cabal-hashes" { } "tar xf ${all-cabal-hashes} --strip-components=1 --one-top-level=$out"' +unpacked_hackage="$(nix-build -E "$extraction_derivation" --no-out-link)" +config_dir=pkgs/development/haskell-modules/configuration-hackage2nix + +echo "Starting hackage2nix to regenerate pkgs/development/haskell-modules/hackage-packages.nix ..." +"$HACKAGE2NIX" \ + --hackage "$unpacked_hackage" \ + --preferred-versions <(for n in "$unpacked_hackage"/*/preferred-versions; do cat "$n"; echo; done) \ + --nixpkgs "$PWD" \ + --config "$config_dir/main.yaml" \ + --config "$config_dir/stackage.yaml" \ + --config "$config_dir/broken.yaml" \ + --config "$config_dir/transitive-broken.yaml" + +if [[ "${1:-}" == "--do-commit" ]]; then +git add pkgs/development/haskell-modules/hackage-packages.nix +git commit -F - << EOF +haskellPackages: regenerate package set based on current config + +This commit has been generated by maintainers/scripts/haskell/regenerate-hackage-packages.sh +EOF +fi + +echo "Regeneration of hackage-packages.nix finished." diff --git a/maintainers/scripts/haskell/regenerate-transitive-broken-packages.sh b/maintainers/scripts/haskell/regenerate-transitive-broken-packages.sh new file mode 100755 index 00000000000..94104e00edb --- /dev/null +++ b/maintainers/scripts/haskell/regenerate-transitive-broken-packages.sh @@ -0,0 +1,15 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p coreutils nix gnused -I nixpkgs=. + +config_file=pkgs/development/haskell-modules/configuration-hackage2nix/transitive-broken.yaml + +cat > $config_file << EOF +# This file is automatically generated by +# maintainers/scripts/haskell/regenerate-transitive-broken-packages.sh +# It is supposed to list all haskellPackages that cannot evaluate because they +# depend on a dependency marked as broken. +dont-distribute-packages: +EOF + +echo "Regenerating list of transitive broken packages ..." +echo -e $(nix-instantiate --eval --strict maintainers/scripts/haskell/transitive-broken-packages.nix) | sed 's/\"//' | LC_ALL=C.UTF-8 sort -i >> $config_file diff --git a/maintainers/scripts/haskell/transitive-broken-packages.nix b/maintainers/scripts/haskell/transitive-broken-packages.nix new file mode 100644 index 00000000000..d4ddaa95765 --- /dev/null +++ b/maintainers/scripts/haskell/transitive-broken-packages.nix @@ -0,0 +1,16 @@ +let + nixpkgs = import ../../..; + inherit (nixpkgs {}) pkgs lib; + getEvaluating = x: + builtins.attrNames ( + lib.filterAttrs ( + _: v: (builtins.tryEval (v.outPath or null)).success && lib.isDerivation v && !v.meta.broken + ) x + ); + brokenDeps = lib.subtractLists + (getEvaluating pkgs.haskellPackages) + (getEvaluating (nixpkgs { config.allowBroken = true; }).haskellPackages); +in +'' + ${lib.concatMapStringsSep "\n" (x: " - ${x}") brokenDeps} +'' diff --git a/maintainers/scripts/haskell/update-cabal2nix-unstable.sh b/maintainers/scripts/haskell/update-cabal2nix-unstable.sh new file mode 100755 index 00000000000..41583704560 --- /dev/null +++ b/maintainers/scripts/haskell/update-cabal2nix-unstable.sh @@ -0,0 +1,17 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p coreutils curl jq gnused haskellPackages.cabal2nix-unstable -I nixpkgs=. + +# Updates cabal2nix-unstable to the latest master of the nixos/cabal2nix repository. +# See regenerate-hackage-packages.sh for details on the purpose of this script. + +set -euo pipefail + +# fetch current master HEAD from Github +head_info="$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/NixOS/cabal2nix/branches/master)" +# extract commit hash +commit="$(jq -r .commit.sha <<< "$head_info")" +# extract commit timestamp and convert to date +date="$(date "--date=$(jq -r .commit.commit.committer.date <<< "$head_info")" +%F)" +# generate nix expression from cabal file, replacing the version with the commit date +echo '# This file defines cabal2nix-unstable, used by maintainers/scripts/haskell/regenerate-hackage-packages.sh.' > pkgs/development/haskell-modules/cabal2nix-unstable.nix +cabal2nix "https://github.com/NixOS/cabal2nix/archive/$commit.tar.gz" | sed -e 's/version = ".*"/version = "'"unstable-$date"'"/' >> pkgs/development/haskell-modules/cabal2nix-unstable.nix diff --git a/maintainers/scripts/haskell/update-hackage.sh b/maintainers/scripts/haskell/update-hackage.sh new file mode 100755 index 00000000000..a7cfecbbb0f --- /dev/null +++ b/maintainers/scripts/haskell/update-hackage.sh @@ -0,0 +1,35 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p nix curl jq nix-prefetch-github git gnused -I nixpkgs=. + +# See regenerate-hackage-packages.sh for details on the purpose of this script. + +set -euo pipefail + +pin_file=pkgs/data/misc/hackage/pin.json +current_commit="$(jq -r .commit $pin_file)" +old_date="$(jq -r .msg $pin_file | sed 's/Update from Hackage at //')" +git_info="$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/commercialhaskell/all-cabal-hashes/branches/hackage)" +head_commit="$(echo "$git_info" | jq -r .commit.sha)" +commit_msg="$(echo "$git_info" | jq -r .commit.commit.message)" +new_date="$(echo "$commit_msg" | sed 's/Update from Hackage at //')" + +if [ "$current_commit" != "$head_commit" ]; then + url="https://github.com/commercialhaskell/all-cabal-hashes/archive/$head_commit.tar.gz" + hash="$(nix-prefetch-url "$url")" + jq -n \ + --arg commit "$head_commit" \ + --arg hash "$hash" \ + --arg url "$url" \ + --arg commit_msg "$commit_msg" \ + '{commit: $commit, url: $url, sha256: $hash, msg: $commit_msg}' \ + > $pin_file +fi + +if [[ "${1:-}" == "--do-commit" ]]; then +git add pkgs/data/misc/hackage/pin.json +git commit -F - << EOF +all-cabal-hashes: $old_date -> $new_date + +This commit has been generated by maintainers/scripts/haskell/update-hackage.sh +EOF +fi diff --git a/maintainers/scripts/haskell/update-stackage.sh b/maintainers/scripts/haskell/update-stackage.sh new file mode 100755 index 00000000000..ecf38dc4b90 --- /dev/null +++ b/maintainers/scripts/haskell/update-stackage.sh @@ -0,0 +1,57 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p nix curl jq nix-prefetch-github git gnused gnugrep -I nixpkgs=. + +set -eu -o pipefail + +tmpfile=$(mktemp "update-stackage.XXXXXXX") +# shellcheck disable=SC2064 + +stackage_config="pkgs/development/haskell-modules/configuration-hackage2nix/stackage.yaml" + +trap "rm ${tmpfile} ${tmpfile}.new" 0 +touch "$tmpfile" "$tmpfile.new" # Creating files here so that trap creates no errors. + +curl -L -s "https://stackage.org/lts/cabal.config" >"$tmpfile" +old_version=$(grep "# Stackage" $stackage_config | sed -E 's/.*([0-9]{2}\.[0-9]+)/\1/') +version=$(sed -rn "s/^--.*http:..(www.)?stackage.org.snapshot.lts-//p" "$tmpfile") + +if [[ "$old_version" == "$version" ]]; then + echo "No new stackage version" + exit 0 # Nothing to do +fi + +echo "Updating Stackage LTS from $old_version to $version." + +# Create a simple yaml version of the file. +sed -r \ + -e '/^--/d' \ + -e 's|^constraints:||' \ + -e 's|^ +| - |' \ + -e 's|,$||' \ + -e '/installed$/d' \ + -e '/^$/d' \ + < "${tmpfile}" | sort --ignore-case >"${tmpfile}.new" + +cat > $stackage_config << EOF +# Stackage LTS $version +# This file is auto-generated by +# maintainers/scripts/haskell/update-stackage.sh +default-package-overrides: +EOF + +# Drop restrictions on some tools where we always want the latest version. +sed -r \ + -e '/ cabal2nix /d' \ + -e '/ distribution-nixpkgs /d' \ + -e '/ jailbreak-cabal /d' \ + -e '/ language-nix /d' \ + < "${tmpfile}.new" >> $stackage_config + +if [[ "${1:-}" == "--do-commit" ]]; then +git add $stackage_config +git commit -F - << EOF +haskellPackages: stackage-lts $old_version -> $version + +This commit has been generated by maintainers/scripts/haskell/update-stackage.sh +EOF +fi diff --git a/maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh b/maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh new file mode 100755 index 00000000000..c49ea68fbfb --- /dev/null +++ b/maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh @@ -0,0 +1,21 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p nix curl gnused -I nixpkgs=. + +# On Hackage every package description shows a category "Distributions" which +# lists a "NixOS" version. +# This script uploads a csv to hackage which will update the displayed versions +# based on the current versions in nixpkgs. This happens with a simple http +# request. + +# For authorization you just need to have any valid hackage account. This +# script uses the `username` and `password-command` field from your +# ~/.cabal/config file. + +# e.g. username: maralorn +# password-command: pass hackage.haskell.org (this can be any command, but not an arbitrary shell expression.) +# Those fields are specified under `upload` on the `cabal` man page. + +package_list="$(nix-build -A haskell.package-list)/nixos-hackage-packages.csv" +username=$(grep "^username:" ~/.cabal/config | sed "s/^username: //") +password_command=$(grep "^password-command:" ~/.cabal/config | sed "s/^password-command: //") +curl -u "$username:$($password_command)" --digest -H "Content-type: text/csv" -T "$package_list" http://hackage.haskell.org/distro/NixOS/packages.csv |