Merge branch 'nixpkgs-update' into master

5 files changed, 303 insertions, 71 deletions

diff --git a/pkgs/games/factorio/default.nix b/pkgs/games/factorio/default.nix
index 0226acf1604..9798d659322 100644
--- a/pkgs/games/factorio/default.nix
+++ b/pkgs/games/factorio/default.nix
@@ -1,5 +1,5 @@
-{ stdenv, fetchurl, makeWrapper, makeDesktopItem
-, alsaLib, libpulseaudio, libX11, libXcursor, libXinerama, libXrandr, libXi, libGL
+{ lib, stdenv, fetchurl, makeWrapper, makeDesktopItem
+, alsa-lib, libpulseaudio, libX11, libXcursor, libXinerama, libXrandr, libXi, libGL
 , libSM, libICE, libXext, factorio-utils
 , releaseType
 , mods ? []
@@ -13,6 +13,8 @@ assert releaseType == "alpha"
 
 let
 
+  inherit (lib) importJSON;
+
   helpMsg = ''
 
     ===FETCH FAILED===
@@ -40,7 +42,7 @@ let
 
       releaseType=alpha
       version=0.17.74
-      nix-prefetch-url file://$HOME/Downloads/factorio_\''${releaseType}_x64_\''${version}.tar.xz --name factorio_\''${releaseType}_x64-\''${version}.tar.xz
+      nix-prefetch-url file://\''$HOME/Downloads/factorio_\''${releaseType}_x64_\''${version}.tar.xz --name factorio_\''${releaseType}_x64-\''${version}.tar.xz
 
     Note the ultimate "_" is replaced with "-" in the --name arg!
   '';
@@ -59,67 +61,54 @@ let
 
   # NB `experimental` directs us to take the latest build, regardless of its branch;
   # hence the (stable, experimental) pairs may sometimes refer to the same distributable.
-  binDists = {
-    x86_64-linux = let bdist = bdistForArch { inUrl = "linux64"; inTar = "x64"; }; in {
-      alpha = {
-        stable        = bdist { sha256 = "0zixscff0svpb0yg8nzczp2z4filqqxi1k0z0nrpzn2hhzhf1464"; version = "1.0.0"; withAuth = true; };
-        experimental  = bdist { sha256 = "0zixscff0svpb0yg8nzczp2z4filqqxi1k0z0nrpzn2hhzhf1464"; version = "1.0.0"; withAuth = true; };
-      };
-      headless = {
-        stable        = bdist { sha256 = "0r0lplns8nxna2viv8qyx9mp4cckdvx6k20w2g2fwnj3jjmf3nc1"; version = "1.0.0"; };
-        experimental  = bdist { sha256 = "0r0lplns8nxna2viv8qyx9mp4cckdvx6k20w2g2fwnj3jjmf3nc1"; version = "1.0.0"; };
-      };
-      demo = {
-        stable        = bdist { sha256 = "0h9cqbp143w47zcl4qg4skns4cngq0k40s5jwbk0wi5asjz8whqn"; version = "1.0.0"; };
-      };
-    };
-    i686-linux = let bdist = bdistForArch { inUrl = "linux32"; inTar = "i386"; }; in {
-      alpha = {
-        stable        = bdist { sha256 = "0nnfkxxqnywx1z05xnndgh71gp4izmwdk026nnjih74m2k5j086l"; version = "0.14.23"; withAuth = true; nameMut = asGz; };
-      };
-    };
-  };
+  versions = importJSON ./versions.json;
+  binDists = makeBinDists versions;
 
   actual = binDists.${stdenv.hostPlatform.system}.${releaseType}.${branch} or (throw "Factorio ${releaseType}-${branch} binaries for ${stdenv.hostPlatform.system} are not available for download.");
 
-  bdistForArch = arch: { version
-                       , sha256
-                       , withAuth ? false
-                       , nameMut ? x: x
-                       }:
-    let
-      url = "https://factorio.com/get-download/${version}/${releaseType}/${arch.inUrl}";
-      name = nameMut "factorio_${releaseType}_${arch.inTar}-${version}.tar.xz";
-    in {
-      inherit version arch;
-      src =
-        if withAuth then
-          (stdenv.lib.overrideDerivation
-            (fetchurl {
-              inherit name url sha256;
-              curlOpts = [
-                "--get"
-                "--data-urlencode" "username@username"
-                "--data-urlencode" "token@token"
-              ];
-            })
-            (_: { # This preHook hides the credentials from /proc
-                  preHook = ''
+  makeBinDists = versions:
+    let f = path: name: value:
+      if builtins.isAttrs value then
+        if value ? "name" then
+          makeBinDist value
+        else
+          builtins.mapAttrs (f (path ++ [ name ])) value
+      else
+        throw "expected attrset at ${toString path} - got ${toString value}";
+    in
+      builtins.mapAttrs (f []) versions;
+  makeBinDist = { name, version, tarDirectory, url, sha256, needsAuth }: {
+    inherit version tarDirectory;
+    src =
+      if !needsAuth then
+        fetchurl { inherit name url sha256; }
+      else
+        (lib.overrideDerivation
+          (fetchurl {
+            inherit name url sha256;
+            curlOpts = [
+              "--get"
+              "--data-urlencode" "username@username"
+              "--data-urlencode" "token@token"
+            ];
+          })
+          (_: { # This preHook hides the credentials from /proc
+                preHook =
+                  if username != "" && token != "" then ''
                     echo -n "${username}" >username
                     echo -n "${token}"    >token
+                  '' else ''
+                    # Deliberately failing since username/token was not provided, so we can't fetch.
+                    # We can't use builtins.throw since we want the result to be used if the tar is in the store already.
+                    exit 1
                   '';
-                  failureHook = ''
-                    cat <<EOF
-                    ${helpMsg}
-                    EOF
-                  '';
-            })
-          )
-        else
-          fetchurl { inherit name url sha256; };
-    };
-
-  asGz = builtins.replaceStrings [".xz"] [".gz"];
+                failureHook = ''
+                  cat <<EOF
+                  ${helpMsg}
+                  EOF
+                '';
+              }));
+  };
 
   configBaseCfg = ''
     use-system-read-write-data-directories=false
@@ -154,12 +143,16 @@ let
     installPhase = ''
       mkdir -p $out/{bin,share/factorio}
       cp -a data $out/share/factorio
-      cp -a bin/${arch.inTar}/factorio $out/bin/factorio
+      cp -a bin/${tarDirectory}/factorio $out/bin/factorio
       patchelf \
         --set-interpreter $(cat $NIX_CC/nix-support/dynamic-linker) \
         $out/bin/factorio
     '';
 
+    passthru.updateScript = if (username != "" && token != "") then [
+      ./update.py "--username=${username}" "--token=${token}"
+    ] else null;
+
     meta = {
       description = "A game in which you build and maintain factories";
       longDescription = ''
@@ -171,13 +164,13 @@ let
         ingenious structures, apply management skills to keep it working and
         finally protect it from the creatures who don't really like you.
 
-        Factorio has been in development since spring of 2012 and it is
-        currently in late alpha.
+        Factorio has been in development since spring of 2012, and reached
+        version 1.0 in mid 2020.
       '';
       homepage = "https://www.factorio.com/";
-      license = stdenv.lib.licenses.unfree;
-      maintainers = with stdenv.lib.maintainers; [ Baughn elitak erictapen priegger ];
-      platforms = [ "i686-linux" "x86_64-linux" ];
+      license = lib.licenses.unfree;
+      maintainers = with lib.maintainers; [ Baughn elitak erictapen priegger lukegb ];
+      platforms = [ "x86_64-linux" ];
     };
   };
 
@@ -185,10 +178,11 @@ let
     headless = base;
     demo = base // {
 
-      buildInputs = [ makeWrapper libpulseaudio ];
+      nativeBuildInputs = [ makeWrapper ];
+      buildInputs = [ libpulseaudio ];
 
-      libPath = stdenv.lib.makeLibraryPath [
-        alsaLib
+      libPath = lib.makeLibraryPath [
+        alsa-lib
         libpulseaudio
         libX11
         libXcursor
diff --git a/pkgs/games/factorio/mods.nix b/pkgs/games/factorio/mods.nix
index 5c9b4f0628a..7327b9e2287 100644
--- a/pkgs/games/factorio/mods.nix
+++ b/pkgs/games/factorio/mods.nix
@@ -2,12 +2,12 @@
 # mods. It will eventually be replaced by a nixos-channel that will provide
 # derivations for most or all of the mods tracked through the official mod
 # manager site.
-{ stdenv, fetchurl
+{ lib, fetchurl
 , factorio-utils
 , allRecommendedMods ? true
 , allOptionalMods ? false
 }:
-with stdenv.lib;
+with lib;
 let
   modDrv = factorio-utils.modDrv { inherit allRecommendedMods allOptionalMods; };
 in
diff --git a/pkgs/games/factorio/update.py b/pkgs/games/factorio/update.py
new file mode 100755
index 00000000000..3806ba3614f
--- /dev/null
+++ b/pkgs/games/factorio/update.py
@@ -0,0 +1,180 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i python -p "python3.withPackages (ps: with ps; [ ps.absl-py ps.requests ])" nix
+
+from collections import defaultdict
+import copy
+from dataclasses import dataclass
+import json
+import os.path
+import subprocess
+from typing import Callable, Dict
+
+from absl import app
+from absl import flags
+from absl import logging
+import requests
+
+
+FACTORIO_API = "https://factorio.com/api/latest-releases"
+
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string('username', '', 'Factorio username for retrieving binaries.')
+flags.DEFINE_string('token', '', 'Factorio token for retrieving binaries.')
+flags.DEFINE_string('out', '', 'Output path for versions.json.')
+
+
+@dataclass
+class System:
+    nix_name: str
+    url_name: str
+    tar_name: str
+
+
+@dataclass
+class ReleaseType:
+    name: str
+    needs_auth: bool = False
+
+
+@dataclass
+class ReleaseChannel:
+    name: str
+
+
+FactorioVersionsJSON = Dict[str, Dict[str, str]]
+OurVersionJSON = Dict[str, Dict[str, Dict[str, Dict[str, str]]]]
+
+
+SYSTEMS = [
+    System(nix_name="x86_64-linux", url_name="linux64", tar_name="x64"),
+]
+
+RELEASE_TYPES = [
+    ReleaseType("alpha", needs_auth=True),
+    ReleaseType("demo"),
+    ReleaseType("headless"),
+]
+
+RELEASE_CHANNELS = [
+    ReleaseChannel("experimental"),
+    ReleaseChannel("stable"),
+]
+
+
+def find_versions_json() -> str:
+    if FLAGS.out:
+        return out
+    try_paths = ["pkgs/games/factorio/versions.json", "versions.json"]
+    for path in try_paths:
+        if os.path.exists(path):
+            return path
+    raise Exception("Couldn't figure out where to write versions.json; try specifying --out")
+
+
+def fetch_versions() -> FactorioVersionsJSON:
+    return json.loads(requests.get("https://factorio.com/api/latest-releases").text)
+
+
+def generate_our_versions(factorio_versions: FactorioVersionsJSON) -> OurVersionJSON:
+    rec_dd = lambda: defaultdict(rec_dd)
+    output = rec_dd()
+
+    # Deal with times where there's no experimental version
+    for rc in RELEASE_CHANNELS:
+        if not factorio_versions[rc.name]:
+            factorio_versions[rc.name] = factorio_versions['stable']
+
+    for system in SYSTEMS:
+        for release_type in RELEASE_TYPES:
+            for release_channel in RELEASE_CHANNELS:
+                version = factorio_versions[release_channel.name][release_type.name]
+                this_release = {
+                    "name":         f"factorio_{release_type.name}_{system.tar_name}-{version}.tar.xz",
+                    "url":          f"https://factorio.com/get-download/{version}/{release_type.name}/{system.url_name}",
+                    "version":      version,
+                    "needsAuth":    release_type.needs_auth,
+                    "tarDirectory": system.tar_name,
+                }
+                output[system.nix_name][release_type.name][release_channel.name] = this_release
+    return output
+
+
+def iter_version(versions: OurVersionJSON, it: Callable[[str, str, str, Dict[str, str]], Dict[str, str]]) -> OurVersionJSON:
+    versions = copy.deepcopy(versions)
+    for system_name, system in versions.items():
+        for release_type_name, release_type in system.items():
+            for release_channel_name, release in release_type.items():
+                release_type[release_channel_name] = it(system_name, release_type_name, release_channel_name, dict(release))
+    return versions
+
+
+def merge_versions(old: OurVersionJSON, new: OurVersionJSON) -> OurVersionJSON:
+    """Copies already-known hashes from version.json to avoid having to re-fetch."""
+    def _merge_version(system_name: str, release_type_name: str, release_channel_name: str, release: Dict[str, str]) -> Dict[str, str]:
+        old_system = old.get(system_name, {})
+        old_release_type = old_system.get(release_type_name, {})
+        old_release = old_release_type.get(release_channel_name, {})
+        if not "sha256" in old_release:
+            logging.info("%s/%s/%s: not copying sha256 since it's missing", system_name, release_type_name, release_channel_name)
+            return release
+        if not all(old_release.get(k, None) == release[k] for k in ['name', 'version', 'url']):
+            logging.info("%s/%s/%s: not copying sha256 due to mismatch", system_name, release_type_name, release_channel_name)
+            return release
+        release["sha256"] = old_release["sha256"]
+        return release
+    return iter_version(new, _merge_version)
+
+
+def nix_prefetch_url(name: str, url: str, algo: str = 'sha256') -> str:
+    cmd = ['nix-prefetch-url', '--type', algo, '--name', name, url]
+    logging.info('running %s', cmd)
+    out = subprocess.check_output(cmd)
+    return out.decode('utf-8').strip()
+
+
+def fill_in_hash(versions: OurVersionJSON) -> OurVersionJSON:
+    """Fill in sha256 hashes for anything missing them."""
+    urls_to_hash = {}
+    def _fill_in_hash(system_name: str, release_type_name: str, release_channel_name: str, release: Dict[str, str]) -> Dict[str, str]:
+        if "sha256" in release:
+            logging.info("%s/%s/%s: skipping fetch, sha256 already present", system_name, release_type_name, release_channel_name)
+            return release
+        url = release["url"]
+        if url in urls_to_hash:
+            logging.info("%s/%s/%s: found url %s in cache", system_name, release_type_name, release_channel_name, url)
+            release["sha256"] = urls_to_hash[url]
+            return release
+        logging.info("%s/%s/%s: fetching %s", system_name, release_type_name, release_channel_name, url)
+        if release["needsAuth"]:
+            if not FLAGS.username or not FLAGS.token:
+                raise Exception("fetching %s/%s/%s from %s requires --username and --token" % (system_name, release_type_name, release_channel_name, url))
+            url += f"?username={FLAGS.username}&token={FLAGS.token}"
+        release["sha256"] = nix_prefetch_url(release["name"], url)
+        urls_to_hash[url] = release["sha256"]
+        return release
+    return iter_version(versions, _fill_in_hash)
+
+
+def main(argv):
+    factorio_versions = fetch_versions()
+    new_our_versions = generate_our_versions(factorio_versions)
+    old_our_versions = None
+    our_versions_path = find_versions_json()
+    if our_versions_path:
+        logging.info('Loading old versions.json from %s', our_versions_path)
+        with open(our_versions_path, 'r') as f:
+            old_our_versions = json.load(f)
+    if old_our_versions:
+        logging.info('Merging in old hashes')
+        new_our_versions = merge_versions(old_our_versions, new_our_versions)
+    logging.info('Fetching necessary tars to get hashes')
+    new_our_versions = fill_in_hash(new_our_versions)
+    with open(our_versions_path, 'w') as f:
+        logging.info('Writing versions.json to %s', our_versions_path)
+        json.dump(new_our_versions, f, sort_keys=True, indent=2)
+        f.write("\n")
+
+if __name__ == '__main__':
+    app.run(main)
diff --git a/pkgs/games/factorio/utils.nix b/pkgs/games/factorio/utils.nix
index 2764592a324..be5b7fd5ecc 100644
--- a/pkgs/games/factorio/utils.nix
+++ b/pkgs/games/factorio/utils.nix
@@ -1,7 +1,7 @@
 # This file provides a top-level function that will be used by both nixpkgs and nixos
 # to generate mod directories for use at runtime by factorio.
-{ stdenv }:
-with stdenv.lib;
+{ lib, stdenv }:
+with lib;
 {
   mkModDirDrv = mods: # a list of mod derivations
     let
diff --git a/pkgs/games/factorio/versions.json b/pkgs/games/factorio/versions.json
new file mode 100644
index 00000000000..500181ff70e
--- /dev/null
+++ b/pkgs/games/factorio/versions.json
@@ -0,0 +1,58 @@
+{
+  "x86_64-linux": {
+    "alpha": {
+      "experimental": {
+        "name": "factorio_alpha_x64-1.1.36.tar.xz",
+        "needsAuth": true,
+        "sha256": "1x9a2lv6zbqawqlxg8bcbx04hjy0pq40macfa4sqi8w6h14wgww8",
+        "tarDirectory": "x64",
+        "url": "https://factorio.com/get-download/1.1.36/alpha/linux64",
+        "version": "1.1.36"
+      },
+      "stable": {
+        "name": "factorio_alpha_x64-1.1.36.tar.xz",
+        "needsAuth": true,
+        "sha256": "1x9a2lv6zbqawqlxg8bcbx04hjy0pq40macfa4sqi8w6h14wgww8",
+        "tarDirectory": "x64",
+        "url": "https://factorio.com/get-download/1.1.36/alpha/linux64",
+        "version": "1.1.36"
+      }
+    },
+    "demo": {
+      "experimental": {
+        "name": "factorio_demo_x64-1.1.35.tar.xz",
+        "needsAuth": false,
+        "sha256": "0yqb4gf2avpxr4vwafws9pv74xyd9g84zggfikfc801ldc7sp29f",
+        "tarDirectory": "x64",
+        "url": "https://factorio.com/get-download/1.1.35/demo/linux64",
+        "version": "1.1.35"
+      },
+      "stable": {
+        "name": "factorio_demo_x64-1.1.36.tar.xz",
+        "needsAuth": false,
+        "sha256": "15fl4pza7n107rrmmdm26kkc12fnrmpn6rjb4ampgzqzn1fq854s",
+        "tarDirectory": "x64",
+        "url": "https://factorio.com/get-download/1.1.36/demo/linux64",
+        "version": "1.1.36"
+      }
+    },
+    "headless": {
+      "experimental": {
+        "name": "factorio_headless_x64-1.1.36.tar.xz",
+        "needsAuth": false,
+        "sha256": "1s8g030xp5nrlmnn21frrd8n4nd7jjmb5hbpj1vhxjrk6vpijh24",
+        "tarDirectory": "x64",
+        "url": "https://factorio.com/get-download/1.1.36/headless/linux64",
+        "version": "1.1.36"
+      },
+      "stable": {
+        "name": "factorio_headless_x64-1.1.36.tar.xz",
+        "needsAuth": false,
+        "sha256": "1s8g030xp5nrlmnn21frrd8n4nd7jjmb5hbpj1vhxjrk6vpijh24",
+        "tarDirectory": "x64",
+        "url": "https://factorio.com/get-download/1.1.36/headless/linux64",
+        "version": "1.1.36"
+      }
+    }
+  }
+}