From 0c10b2baa6bf61c8ddaed7cdb6c2f2dbaab42662 Mon Sep 17 00:00:00 2001 From: Allan Espinosa Date: Sun, 20 May 2018 21:09:31 -0400 Subject: nixos/hadoop: add hadoop module (hdfs, yarn) --- nixos/modules/misc/ids.nix | 8 +++ nixos/modules/module-list.nix | 1 + nixos/modules/services/cluster/hadoop/conf.nix | 31 ++++++++++ nixos/modules/services/cluster/hadoop/default.nix | 63 +++++++++++++++++++ nixos/modules/services/cluster/hadoop/hdfs.nix | 73 ++++++++++++++++++++++ nixos/modules/services/cluster/hadoop/yarn.nix | 74 +++++++++++++++++++++++ 6 files changed, 250 insertions(+) create mode 100644 nixos/modules/services/cluster/hadoop/conf.nix create mode 100644 nixos/modules/services/cluster/hadoop/default.nix create mode 100644 nixos/modules/services/cluster/hadoop/hdfs.nix create mode 100644 nixos/modules/services/cluster/hadoop/yarn.nix (limited to 'nixos/modules') diff --git a/nixos/modules/misc/ids.nix b/nixos/modules/misc/ids.nix index 73231edf077..aac86087f9e 100644 --- a/nixos/modules/misc/ids.nix +++ b/nixos/modules/misc/ids.nix @@ -317,6 +317,10 @@ restic = 291; openvpn = 292; meguca = 293; + yarn = 294; + hdfs = 295; + mapred = 296; + hadoop = 297; # When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399! @@ -594,6 +598,10 @@ restic = 291; openvpn = 292; meguca = 293; + yarn = 294; + hdfs = 295; + mapred = 296; + hadoop = 297; # When adding a gid, make sure it doesn't match an existing # uid. Users and groups with the same name should have equal diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index c425f3c6507..5ed01a8da40 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -178,6 +178,7 @@ ./services/backup/rsnapshot.nix ./services/backup/tarsnap.nix ./services/backup/znapzend.nix + ./services/cluster/hadoop/default.nix ./services/cluster/kubernetes/default.nix ./services/cluster/kubernetes/dns.nix ./services/cluster/kubernetes/dashboard.nix diff --git a/nixos/modules/services/cluster/hadoop/conf.nix b/nixos/modules/services/cluster/hadoop/conf.nix new file mode 100644 index 00000000000..38db10406b9 --- /dev/null +++ b/nixos/modules/services/cluster/hadoop/conf.nix @@ -0,0 +1,31 @@ +{ hadoop, pkgs }: +let + propertyXml = name: value: '' + + ${name} + ${builtins.toString value} + + ''; + siteXml = fileName: properties: pkgs.writeTextDir fileName '' + + + + ${builtins.concatStringsSep "\n" (pkgs.lib.mapAttrsToList propertyXml properties)} + + ''; + userFunctions = '' + hadoop_verify_logdir() { + echo Skipping verification of log directory + } + ''; +in +pkgs.buildEnv { + name = "hadoop-conf"; + paths = [ + (siteXml "core-site.xml" hadoop.coreSite) + (siteXml "hdfs-site.xml" hadoop.hdfsSite) + (siteXml "mapred-site.xml" hadoop.mapredSite) + (siteXml "yarn-site.xml" hadoop.yarnSite) + (pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions) + ]; +} diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix new file mode 100644 index 00000000000..53c13fd0603 --- /dev/null +++ b/nixos/modules/services/cluster/hadoop/default.nix @@ -0,0 +1,63 @@ +{ config, lib, pkgs, ...}: +let + cfg = config.services.hadoop; + hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; }; +in +with lib; +{ + imports = [ ./yarn.nix ./hdfs.nix ]; + + options.services.hadoop = { + coreSite = mkOption { + default = {}; + example = { + "fs.defaultFS" = "hdfs://localhost"; + }; + description = "Hadoop core-site.xml definition"; + }; + + hdfsSite = mkOption { + default = {}; + example = { + "dfs.nameservices" = "namenode1"; + }; + description = "Hadoop hdfs-site.xml definition"; + }; + + mapredSite = mkOption { + default = {}; + example = { + "mapreduce.map.cpu.vcores" = "1"; + }; + description = "Hadoop mapred-site.xml definition"; + }; + + yarnSite = mkOption { + default = {}; + example = { + "yarn.resourcemanager.ha.id" = "resourcemanager1"; + }; + description = "Hadoop yarn-site.xml definition"; + }; + + package = mkOption { + type = types.package; + default = pkgs.hadoop; + defaultText = "pkgs.hadoop"; + example = literalExample "pkgs.hadoop"; + description = '' + ''; + }; + }; + + + config = mkMerge [ + (mkIf (builtins.hasAttr "yarn" config.users.extraUsers || + builtins.hasAttr "hdfs" config.users.extraUsers ) { + users.extraGroups.hadoop = { + gid = config.ids.gids.hadoop; + }; + }) + + ]; +} diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix new file mode 100644 index 00000000000..48020e6139c --- /dev/null +++ b/nixos/modules/services/cluster/hadoop/hdfs.nix @@ -0,0 +1,73 @@ +{ config, lib, pkgs, ...}: +let + cfg = config.services.hadoop; + hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; }; +in +with lib; +{ + options.services.hadoop.hdfs = { + namenode.enabled = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run the Hadoop YARN NameNode + ''; + }; + datanode.enabled = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run the Hadoop YARN DataNode + ''; + }; + }; + + config = mkMerge [ + (mkIf cfg.hdfs.namenode.enabled { + systemd.services."hdfs-namenode" = { + description = "Hadoop HDFS NameNode"; + wantedBy = [ "multi-user.target" ]; + + environment = { + HADOOP_HOME = "${cfg.package}"; + }; + + preStart = '' + ${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true + ''; + + serviceConfig = { + User = "hdfs"; + SyslogIdentifier = "hdfs-namenode"; + ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} namenode"; + }; + }; + }) + (mkIf cfg.hdfs.datanode.enabled { + systemd.services."hdfs-datanode" = { + description = "Hadoop HDFS DataNode"; + wantedBy = [ "multi-user.target" ]; + + environment = { + HADOOP_HOME = "${cfg.package}"; + }; + + serviceConfig = { + User = "hdfs"; + SyslogIdentifier = "hdfs-datanode"; + ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} datanode"; + }; + }; + }) + (mkIf ( + cfg.hdfs.namenode.enabled || cfg.hdfs.datanode.enabled + ) { + users.extraUsers.hdfs = { + description = "Hadoop HDFS user"; + group = "hadoop"; + uid = config.ids.uids.hdfs; + }; + }) + + ]; +} diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix new file mode 100644 index 00000000000..ce5b04a331c --- /dev/null +++ b/nixos/modules/services/cluster/hadoop/yarn.nix @@ -0,0 +1,74 @@ +{ config, lib, pkgs, ...}: +let + cfg = config.services.hadoop; + hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; }; +in +with lib; +{ + options.services.hadoop.yarn = { + resourcemanager.enabled = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run the Hadoop YARN ResourceManager + ''; + }; + nodemanager.enabled = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run the Hadoop YARN NodeManager + ''; + }; + }; + + config = mkMerge [ + (mkIf ( + cfg.yarn.resourcemanager.enabled || cfg.yarn.nodemanager.enabled + ) { + + users.extraUsers.yarn = { + description = "Hadoop YARN user"; + group = "hadoop"; + uid = config.ids.uids.yarn; + }; + }) + + (mkIf cfg.yarn.resourcemanager.enabled { + systemd.services."yarn-resourcemanager" = { + description = "Hadoop YARN ResourceManager"; + wantedBy = [ "multi-user.target" ]; + + environment = { + HADOOP_HOME = "${cfg.package}"; + }; + + serviceConfig = { + User = "yarn"; + SyslogIdentifier = "yarn-resourcemanager"; + ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " + + " resourcemanager"; + }; + }; + }) + + (mkIf cfg.yarn.nodemanager.enabled { + systemd.services."yarn-nodemanager" = { + description = "Hadoop YARN NodeManager"; + wantedBy = [ "multi-user.target" ]; + + environment = { + HADOOP_HOME = "${cfg.package}"; + }; + + serviceConfig = { + User = "yarn"; + SyslogIdentifier = "yarn-nodemanager"; + ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " + + " nodemanager"; + }; + }; + }) + + ]; +} -- cgit 1.4.1