nixos/hadoop: add hadoop module (hdfs, yarn)

6 files changed, 250 insertions, 0 deletions

diff --git a/nixos/modules/misc/ids.nix b/nixos/modules/misc/ids.nix
index 73231edf077..aac86087f9e 100644
--- a/nixos/modules/misc/ids.nix
+++ b/nixos/modules/misc/ids.nix
@@ -317,6 +317,10 @@
       restic = 291;
       openvpn = 292;
       meguca = 293;
+      yarn = 294;
+      hdfs = 295;
+      mapred = 296;
+      hadoop = 297;
 
       # When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399!
 
@@ -594,6 +598,10 @@
       restic = 291;
       openvpn = 292;
       meguca = 293;
+      yarn = 294;
+      hdfs = 295;
+      mapred = 296;
+      hadoop = 297;
 
       # When adding a gid, make sure it doesn't match an existing
       # uid. Users and groups with the same name should have equal
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index c425f3c6507..5ed01a8da40 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -178,6 +178,7 @@
   ./services/backup/rsnapshot.nix
   ./services/backup/tarsnap.nix
   ./services/backup/znapzend.nix
+  ./services/cluster/hadoop/default.nix
   ./services/cluster/kubernetes/default.nix
   ./services/cluster/kubernetes/dns.nix
   ./services/cluster/kubernetes/dashboard.nix
diff --git a/nixos/modules/services/cluster/hadoop/conf.nix b/nixos/modules/services/cluster/hadoop/conf.nix
new file mode 100644
index 00000000000..38db10406b9
--- /dev/null
+++ b/nixos/modules/services/cluster/hadoop/conf.nix
@@ -0,0 +1,31 @@
+{ hadoop, pkgs }:
+let
+  propertyXml = name: value: ''
+    <property>
+      <name>${name}</name>
+      <value>${builtins.toString value}</value>
+    </property>
+  '';
+  siteXml = fileName: properties: pkgs.writeTextDir fileName ''
+    <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+    <!-- generated by NixOS -->
+    <configuration>
+      ${builtins.concatStringsSep "\n" (pkgs.lib.mapAttrsToList propertyXml properties)}
+    </configuration>
+  '';
+  userFunctions = ''
+    hadoop_verify_logdir() {
+      echo Skipping verification of log directory
+    }
+  '';
+in
+pkgs.buildEnv {
+  name = "hadoop-conf";
+  paths = [
+    (siteXml "core-site.xml" hadoop.coreSite)
+    (siteXml "hdfs-site.xml" hadoop.hdfsSite)
+    (siteXml "mapred-site.xml" hadoop.mapredSite)
+    (siteXml "yarn-site.xml" hadoop.yarnSite)
+    (pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions)
+  ];
+}
diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix
new file mode 100644
index 00000000000..53c13fd0603
--- /dev/null
+++ b/nixos/modules/services/cluster/hadoop/default.nix
@@ -0,0 +1,63 @@
+{ config, lib, pkgs, ...}:
+let 
+  cfg = config.services.hadoop;
+  hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; };
+in
+with lib;
+{
+  imports = [ ./yarn.nix ./hdfs.nix ];
+
+  options.services.hadoop = {
+    coreSite = mkOption {
+      default = {};
+      example = {
+        "fs.defaultFS" = "hdfs://localhost";
+      };
+      description = "Hadoop core-site.xml definition";
+    };
+
+    hdfsSite = mkOption {
+      default = {};
+      example = {
+        "dfs.nameservices" = "namenode1";
+      };
+      description = "Hadoop hdfs-site.xml definition";
+    };
+
+    mapredSite = mkOption {
+      default = {};
+      example = {
+        "mapreduce.map.cpu.vcores" = "1";
+      };
+      description = "Hadoop mapred-site.xml definition";
+    };
+
+    yarnSite = mkOption {
+      default = {};
+      example = {
+        "yarn.resourcemanager.ha.id" = "resourcemanager1";
+      };
+      description = "Hadoop yarn-site.xml definition";
+    };
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.hadoop;
+      defaultText = "pkgs.hadoop";
+      example = literalExample "pkgs.hadoop";
+      description = ''
+      '';
+    };
+  };
+
+
+  config = mkMerge [
+    (mkIf (builtins.hasAttr "yarn" config.users.extraUsers ||
+           builtins.hasAttr "hdfs" config.users.extraUsers ) {
+      users.extraGroups.hadoop = {
+        gid = config.ids.gids.hadoop;
+      };
+    })
+
+  ];
+}
diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix
new file mode 100644
index 00000000000..48020e6139c
--- /dev/null
+++ b/nixos/modules/services/cluster/hadoop/hdfs.nix
@@ -0,0 +1,73 @@
+{ config, lib, pkgs, ...}:
+let
+  cfg = config.services.hadoop;
+  hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; };
+in
+with lib;
+{
+  options.services.hadoop.hdfs = {
+    namenode.enabled = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to run the Hadoop YARN NameNode
+      '';
+    };
+    datanode.enabled = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to run the Hadoop YARN DataNode
+      '';
+    };
+  };
+
+  config = mkMerge [
+    (mkIf cfg.hdfs.namenode.enabled {
+      systemd.services."hdfs-namenode" = {
+        description = "Hadoop HDFS NameNode";
+        wantedBy = [ "multi-user.target" ];
+
+        environment = {
+          HADOOP_HOME = "${cfg.package}";
+        };
+
+        preStart = ''
+          ${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true
+        '';
+
+        serviceConfig = {
+          User = "hdfs";
+          SyslogIdentifier = "hdfs-namenode";
+          ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} namenode";
+        };
+      };
+    })
+    (mkIf cfg.hdfs.datanode.enabled {
+      systemd.services."hdfs-datanode" = {
+        description = "Hadoop HDFS DataNode";
+        wantedBy = [ "multi-user.target" ];
+
+        environment = {
+          HADOOP_HOME = "${cfg.package}";
+        };
+
+        serviceConfig = {
+          User = "hdfs";
+          SyslogIdentifier = "hdfs-datanode";
+          ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} datanode";
+        };
+      };
+    })
+    (mkIf (
+        cfg.hdfs.namenode.enabled || cfg.hdfs.datanode.enabled
+    ) {
+      users.extraUsers.hdfs = {
+        description = "Hadoop HDFS user";
+        group = "hadoop";
+        uid = config.ids.uids.hdfs;
+      };
+    })
+
+  ];
+}
diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix
new file mode 100644
index 00000000000..ce5b04a331c
--- /dev/null
+++ b/nixos/modules/services/cluster/hadoop/yarn.nix
@@ -0,0 +1,74 @@
+{ config, lib, pkgs, ...}:
+let
+  cfg = config.services.hadoop;
+  hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; };
+in
+with lib;
+{
+  options.services.hadoop.yarn = {
+    resourcemanager.enabled = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to run the Hadoop YARN ResourceManager
+      '';
+    };
+    nodemanager.enabled = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to run the Hadoop YARN NodeManager
+      '';
+    };
+  };
+
+  config = mkMerge [
+    (mkIf (
+        cfg.yarn.resourcemanager.enabled || cfg.yarn.nodemanager.enabled
+    ) {
+
+      users.extraUsers.yarn = {
+        description = "Hadoop YARN user";
+        group = "hadoop";
+        uid = config.ids.uids.yarn;
+      };
+    })
+
+    (mkIf cfg.yarn.resourcemanager.enabled {
+      systemd.services."yarn-resourcemanager" = {
+        description = "Hadoop YARN ResourceManager";
+        wantedBy = [ "multi-user.target" ];
+
+        environment = {
+          HADOOP_HOME = "${cfg.package}";
+        };
+
+        serviceConfig = {
+          User = "yarn";
+          SyslogIdentifier = "yarn-resourcemanager";
+          ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +
+                      " resourcemanager";
+        };
+      };
+    })
+
+    (mkIf cfg.yarn.nodemanager.enabled {
+      systemd.services."yarn-nodemanager" = {
+        description = "Hadoop YARN NodeManager";
+        wantedBy = [ "multi-user.target" ];
+
+        environment = {
+          HADOOP_HOME = "${cfg.package}";
+        };
+
+        serviceConfig = {
+          User = "yarn";
+          SyslogIdentifier = "yarn-nodemanager";
+          ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +
+                      " nodemanager";
+        };
+      };
+    })
+
+  ];
+}