factorio: add an updateScript

The updateScript knows how to automatically fetch and update the version
data from the Factorio versions API (and update the hashes
appropriately), which makes it easier to update whenever experimental
does.

3 files changed, 286 insertions, 66 deletions

diff --git a/pkgs/games/factorio/default.nix b/pkgs/games/factorio/default.nix
index f5f12b47dd7..14e00380e87 100644
--- a/pkgs/games/factorio/default.nix
+++ b/pkgs/games/factorio/default.nix
@@ -13,6 +13,8 @@ assert releaseType == "alpha"
 
 let
 
+  inherit (stdenv.lib) importJSON;
+
   helpMsg = ''
 
     ===FETCH FAILED===
@@ -59,72 +61,54 @@ let
 
   # NB `experimental` directs us to take the latest build, regardless of its branch;
   # hence the (stable, experimental) pairs may sometimes refer to the same distributable.
-  binDists = {
-    x86_64-linux = let bdist = bdistForArch { inUrl = "linux64"; inTar = "x64"; }; in {
-      alpha = {
-        stable        = bdist { sha256 = "0zixscff0svpb0yg8nzczp2z4filqqxi1k0z0nrpzn2hhzhf1464"; version = "1.0.0"; withAuth = true; };
-        experimental  = bdist { sha256 = "0cmia16d5dhy3f8mck926d7rrnavxmvb6a72ymjllxm37slsx60j"; version = "1.1.2"; withAuth = true; };
-      };
-      headless = {
-        stable        = bdist { sha256 = "0r0lplns8nxna2viv8qyx9mp4cckdvx6k20w2g2fwnj3jjmf3nc1"; version = "1.0.0"; };
-        experimental  = bdist { sha256 = "0x3lwz11z8cczqr5i799m4yg8x3yk6h5qz48pfzw4l2ikrrwgahd"; version = "1.1.2"; };
-      };
-      demo = {
-        stable        = bdist { sha256 = "0h9cqbp143w47zcl4qg4skns4cngq0k40s5jwbk0wi5asjz8whqn"; version = "1.0.0"; };
-      };
-    };
-    i686-linux = let bdist = bdistForArch { inUrl = "linux32"; inTar = "i386"; }; in {
-      alpha = {
-        stable        = bdist { sha256 = "0nnfkxxqnywx1z05xnndgh71gp4izmwdk026nnjih74m2k5j086l"; version = "0.14.23"; withAuth = true; nameMut = asGz; };
-      };
-    };
-  };
+  versions = importJSON ./versions.json;
+  binDists = makeBinDists versions;
 
   actual = binDists.${stdenv.hostPlatform.system}.${releaseType}.${branch} or (throw "Factorio ${releaseType}-${branch} binaries for ${stdenv.hostPlatform.system} are not available for download.");
 
-  bdistForArch = arch: { version
-                       , sha256
-                       , withAuth ? false
-                       , nameMut ? x: x
-                       }:
-    let
-      url = "https://factorio.com/get-download/${version}/${releaseType}/${arch.inUrl}";
-      name = nameMut "factorio_${releaseType}_${arch.inTar}-${version}.tar.xz";
-    in {
-      inherit version arch;
-      src =
-        if withAuth then
-          (stdenv.lib.overrideDerivation
-            (fetchurl {
-              inherit name url sha256;
-              curlOpts = [
-                "--get"
-                "--data-urlencode" "username@username"
-                "--data-urlencode" "token@token"
-              ];
-            })
-            (_: { # This preHook hides the credentials from /proc
-                  preHook =
-                    if username != "" && token != "" then ''
-                      echo -n "${username}" >username
-                      echo -n "${token}"    >token
-                    '' else ''
-                      # Deliberately failing since username/token was not provided, so we can't fetch.
-                      # We can't use builtins.throw since we want the result to be used if the tar is in the store already.
-                      exit 1
-                    '';
-                  failureHook = ''
-                    cat <<EOF
-                    ${helpMsg}
-                    EOF
-                  '';
-            })
-          )
+  makeBinDists = versions:
+    let f = path: name: value:
+      if builtins.isAttrs value then
+        if value ? "name" then
+          makeBinDist value
         else
-          fetchurl { inherit name url sha256; };
-    };
-
-  asGz = builtins.replaceStrings [".xz"] [".gz"];
+          builtins.mapAttrs (f (path ++ [ name ])) value
+      else
+        throw "expected attrset at ${toString path} - got ${toString value}";
+    in
+      builtins.mapAttrs (f []) versions;
+  makeBinDist = { name, version, tarDirectory, url, sha256, needsAuth }: {
+    inherit version tarDirectory;
+    src =
+      if !needsAuth then
+        fetchurl { inherit name url sha256; }
+      else
+        (stdenv.lib.overrideDerivation
+          (fetchurl {
+            inherit name url sha256;
+            curlOpts = [
+              "--get"
+              "--data-urlencode" "username@username"
+              "--data-urlencode" "token@token"
+            ];
+          })
+          (_: { # This preHook hides the credentials from /proc
+                preHook =
+                  if username != "" && token != "" then ''
+                    echo -n "${username}" >username
+                    echo -n "${token}"    >token
+                  '' else ''
+                    # Deliberately failing since username/token was not provided, so we can't fetch.
+                    # We can't use builtins.throw since we want the result to be used if the tar is in the store already.
+                    exit 1
+                  '';
+                failureHook = ''
+                  cat <<EOF
+                  ${helpMsg}
+                  EOF
+                '';
+              }));
+  };
 
   configBaseCfg = ''
     use-system-read-write-data-directories=false
@@ -159,12 +143,16 @@ let
     installPhase = ''
       mkdir -p $out/{bin,share/factorio}
       cp -a data $out/share/factorio
-      cp -a bin/${arch.inTar}/factorio $out/bin/factorio
+      cp -a bin/${tarDirectory}/factorio $out/bin/factorio
       patchelf \
         --set-interpreter $(cat $NIX_CC/nix-support/dynamic-linker) \
         $out/bin/factorio
     '';
 
+    passthru.updateScript = if (username != "" && token != "") then [
+      ./update.py "--username=${username}" "--token=${token}"
+    ] else null;
+
     meta = {
       description = "A game in which you build and maintain factories";
       longDescription = ''
@@ -176,13 +164,13 @@ let
         ingenious structures, apply management skills to keep it working and
         finally protect it from the creatures who don't really like you.
 
-        Factorio has been in development since spring of 2012 and it is
-        currently in late alpha.
+        Factorio has been in development since spring of 2012, and reached
+        version 1.0 in mid 2020.
       '';
       homepage = "https://www.factorio.com/";
       license = stdenv.lib.licenses.unfree;
       maintainers = with stdenv.lib.maintainers; [ Baughn elitak erictapen priegger ];
-      platforms = [ "i686-linux" "x86_64-linux" ];
+      platforms = [ "x86_64-linux" ];
     };
   };
 
diff --git a/pkgs/games/factorio/update.py b/pkgs/games/factorio/update.py
new file mode 100755
index 00000000000..b1cdce6b4aa
--- /dev/null
+++ b/pkgs/games/factorio/update.py
@@ -0,0 +1,174 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i python -p "python3.withPackages (ps: with ps; [ ps.absl-py ps.requests ])" nix
+
+from collections import defaultdict
+import copy
+from dataclasses import dataclass
+import json
+import os.path
+import subprocess
+from typing import Callable, Dict
+
+from absl import app
+from absl import flags
+from absl import logging
+import requests
+
+
+FACTORIO_API = "https://factorio.com/api/latest-releases"
+
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string('username', '', 'Factorio username for retrieving binaries.')
+flags.DEFINE_string('token', '', 'Factorio token for retrieving binaries.')
+flags.DEFINE_string('out', '', 'Output path for versions.json.')
+
+
+@dataclass
+class System:
+    nix_name: str
+    url_name: str
+    tar_name: str
+
+
+@dataclass
+class ReleaseType:
+    name: str
+    needs_auth: bool = False
+
+
+@dataclass
+class ReleaseChannel:
+    name: str
+
+
+FactorioVersionsJSON = Dict[str, Dict[str, str]]
+OurVersionJSON = Dict[str, Dict[str, Dict[str, Dict[str, str]]]]
+
+
+SYSTEMS = [
+    System(nix_name="x86_64-linux", url_name="linux64", tar_name="x64"),
+]
+
+RELEASE_TYPES = [
+    ReleaseType("alpha", needs_auth=True),
+    ReleaseType("demo"),
+    ReleaseType("headless"),
+]
+
+RELEASE_CHANNELS = [
+    ReleaseChannel("experimental"),
+    ReleaseChannel("stable"),
+]
+
+
+def find_versions_json() -> str:
+    if FLAGS.out:
+        return out
+    try_paths = ["pkgs/games/factorio/versions.json", "versions.json"]
+    for path in try_paths:
+        if os.path.exists(path):
+            return path
+    raise Exception("Couldn't figure out where to write versions.json; try specifying --out")
+
+
+def fetch_versions() -> FactorioVersionsJSON:
+    return json.loads(requests.get("https://factorio.com/api/latest-releases").text)
+
+
+def generate_our_versions(factorio_versions: FactorioVersionsJSON) -> OurVersionJSON:
+    rec_dd = lambda: defaultdict(rec_dd)
+    output = rec_dd()
+    for system in SYSTEMS:
+        for release_type in RELEASE_TYPES:
+            for release_channel in RELEASE_CHANNELS:
+                version = factorio_versions[release_channel.name][release_type.name]
+                this_release = {
+                    "name":         f"factorio_{release_type.name}_{system.tar_name}-{version}.tar.xz",
+                    "url":          f"https://factorio.com/get-download/{version}/{release_type.name}/{system.url_name}",
+                    "version":      version,
+                    "needsAuth":    release_type.needs_auth,
+                    "tarDirectory": system.tar_name,
+                }
+                output[system.nix_name][release_type.name][release_channel.name] = this_release
+    return output
+
+
+def iter_version(versions: OurVersionJSON, it: Callable[[str, str, str, Dict[str, str]], Dict[str, str]]) -> OurVersionJSON:
+    versions = copy.deepcopy(versions)
+    for system_name, system in versions.items():
+        for release_type_name, release_type in system.items():
+            for release_channel_name, release in release_type.items():
+                release_type[release_channel_name] = it(system_name, release_type_name, release_channel_name, dict(release))
+    return versions
+
+
+def merge_versions(old: OurVersionJSON, new: OurVersionJSON) -> OurVersionJSON:
+    """Copies already-known hashes from version.json to avoid having to re-fetch."""
+    def _merge_version(system_name: str, release_type_name: str, release_channel_name: str, release: Dict[str, str]) -> Dict[str, str]:
+        old_system = old.get(system_name, {})
+        old_release_type = old_system.get(release_type_name, {})
+        old_release = old_release_type.get(release_channel_name, {})
+        if not "sha256" in old_release:
+            logging.info("%s/%s/%s: not copying sha256 since it's missing", system_name, release_type_name, release_channel_name)
+            return release
+        if not all(old_release.get(k, None) == release[k] for k in ['name', 'version', 'url']):
+            logging.info("%s/%s/%s: not copying sha256 due to mismatch", system_name, release_type_name, release_channel_name)
+            return release
+        release["sha256"] = old_release["sha256"]
+        return release
+    return iter_version(new, _merge_version)
+
+
+def nix_prefetch_url(name: str, url: str, algo: str = 'sha256') -> str:
+    cmd = ['nix-prefetch-url', '--type', algo, '--name', name, url]
+    logging.info('running %s', cmd)
+    out = subprocess.check_output(cmd)
+    return out.decode('utf-8').strip()
+
+
+def fill_in_hash(versions: OurVersionJSON) -> OurVersionJSON:
+    """Fill in sha256 hashes for anything missing them."""
+    urls_to_hash = {}
+    def _fill_in_hash(system_name: str, release_type_name: str, release_channel_name: str, release: Dict[str, str]) -> Dict[str, str]:
+        if "sha256" in release:
+            logging.info("%s/%s/%s: skipping fetch, sha256 already present", system_name, release_type_name, release_channel_name)
+            return release
+        url = release["url"]
+        if url in urls_to_hash:
+            logging.info("%s/%s/%s: found url %s in cache", system_name, release_type_name, release_channel_name, url)
+            release["sha256"] = urls_to_hash[url]
+            return release
+        logging.info("%s/%s/%s: fetching %s", system_name, release_type_name, release_channel_name, url)
+        if release["needsAuth"]:
+            if not FLAGS.username or not FLAGS.token:
+                raise Exception("fetching %s/%s/%s from %s requires --username and --token" % (system_name, release_type_name, release_channel_name, url))
+            url += f"?username={FLAGS.username}&token={FLAGS.token}"
+        release["sha256"] = nix_prefetch_url(release["name"], url)
+        urls_to_hash[url] = release["sha256"]
+        return release
+    return iter_version(versions, _fill_in_hash)
+
+
+def main(argv):
+    factorio_versions = fetch_versions()
+    new_our_versions = generate_our_versions(factorio_versions)
+    old_our_versions = None
+    our_versions_path = find_versions_json()
+    if our_versions_path:
+        logging.info('Loading old versions.json from %s', our_versions_path)
+        with open(our_versions_path, 'r') as f:
+            old_our_versions = json.load(f)
+    if old_our_versions:
+        logging.info('Merging in old hashes')
+        new_our_versions = merge_versions(old_our_versions, new_our_versions)
+    logging.info('Fetching necessary tars to get hashes')
+    new_our_versions = fill_in_hash(new_our_versions)
+    with open(our_versions_path, 'w') as f:
+        logging.info('Writing versions.json to %s', our_versions_path)
+        json.dump(new_our_versions, f, sort_keys=True, indent=2)
+        f.write("\n")
+
+if __name__ == '__main__':
+    app.run(main)
diff --git a/pkgs/games/factorio/versions.json b/pkgs/games/factorio/versions.json
new file mode 100644
index 00000000000..82ab40ec3cb
--- /dev/null
+++ b/pkgs/games/factorio/versions.json
@@ -0,0 +1,58 @@
+{
+  "x86_64-linux": {
+    "alpha": {
+      "experimental": {
+        "name": "factorio_alpha_x64-1.1.2.tar.xz",
+        "needsAuth": true,
+        "sha256": "0cmia16d5dhy3f8mck926d7rrnavxmvb6a72ymjllxm37slsx60j",
+        "tarDirectory": "x64",
+        "url": "https://factorio.com/get-download/1.1.2/alpha/linux64",
+        "version": "1.1.2"
+      },
+      "stable": {
+        "name": "factorio_alpha_x64-1.0.0.tar.xz",
+        "needsAuth": true,
+        "sha256": "0zixscff0svpb0yg8nzczp2z4filqqxi1k0z0nrpzn2hhzhf1464",
+        "tarDirectory": "x64",
+        "url": "https://factorio.com/get-download/1.0.0/alpha/linux64",
+        "version": "1.0.0"
+      }
+    },
+    "demo": {
+      "experimental": {
+        "name": "factorio_demo_x64-1.0.0.tar.xz",
+        "needsAuth": false,
+        "sha256": "0h9cqbp143w47zcl4qg4skns4cngq0k40s5jwbk0wi5asjz8whqn",
+        "tarDirectory": "x64",
+        "url": "https://factorio.com/get-download/1.0.0/demo/linux64",
+        "version": "1.0.0"
+      },
+      "stable": {
+        "name": "factorio_demo_x64-1.0.0.tar.xz",
+        "needsAuth": false,
+        "sha256": "0h9cqbp143w47zcl4qg4skns4cngq0k40s5jwbk0wi5asjz8whqn",
+        "tarDirectory": "x64",
+        "url": "https://factorio.com/get-download/1.0.0/demo/linux64",
+        "version": "1.0.0"
+      }
+    },
+    "headless": {
+      "experimental": {
+        "name": "factorio_headless_x64-1.1.2.tar.xz",
+        "needsAuth": false,
+        "sha256": "0x3lwz11z8cczqr5i799m4yg8x3yk6h5qz48pfzw4l2ikrrwgahd",
+        "tarDirectory": "x64",
+        "url": "https://factorio.com/get-download/1.1.2/headless/linux64",
+        "version": "1.1.2"
+      },
+      "stable": {
+        "name": "factorio_headless_x64-1.0.0.tar.xz",
+        "needsAuth": false,
+        "sha256": "0r0lplns8nxna2viv8qyx9mp4cckdvx6k20w2g2fwnj3jjmf3nc1",
+        "tarDirectory": "x64",
+        "url": "https://factorio.com/get-download/1.0.0/headless/linux64",
+        "version": "1.0.0"
+      }
+    }
+  }
+}