diff options
Diffstat (limited to 'nixos/modules/services/cluster/hadoop')
-rw-r--r-- | nixos/modules/services/cluster/hadoop/conf.nix | 44 | ||||
-rw-r--r-- | nixos/modules/services/cluster/hadoop/default.nix | 223 | ||||
-rw-r--r-- | nixos/modules/services/cluster/hadoop/hdfs.nix | 204 | ||||
-rw-r--r-- | nixos/modules/services/cluster/hadoop/yarn.nix | 200 |
4 files changed, 671 insertions, 0 deletions
diff --git a/nixos/modules/services/cluster/hadoop/conf.nix b/nixos/modules/services/cluster/hadoop/conf.nix new file mode 100644 index 00000000000..e3c26a0d550 --- /dev/null +++ b/nixos/modules/services/cluster/hadoop/conf.nix @@ -0,0 +1,44 @@ +{ cfg, pkgs, lib }: +let + propertyXml = name: value: lib.optionalString (value != null) '' + <property> + <name>${name}</name> + <value>${builtins.toString value}</value> + </property> + ''; + siteXml = fileName: properties: pkgs.writeTextDir fileName '' + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- generated by NixOS --> + <configuration> + ${builtins.concatStringsSep "\n" (pkgs.lib.mapAttrsToList propertyXml properties)} + </configuration> + ''; + cfgLine = name: value: '' + ${name}=${builtins.toString value} + ''; + cfgFile = fileName: properties: pkgs.writeTextDir fileName '' + # generated by NixOS + ${builtins.concatStringsSep "" (pkgs.lib.mapAttrsToList cfgLine properties)} + ''; + userFunctions = '' + hadoop_verify_logdir() { + echo Skipping verification of log directory + } + ''; + hadoopEnv = '' + export HADOOP_LOG_DIR=/tmp/hadoop/$USER + ''; +in +pkgs.runCommand "hadoop-conf" {} (with cfg; '' + mkdir -p $out/ + cp ${siteXml "core-site.xml" (coreSite // coreSiteInternal)}/* $out/ + cp ${siteXml "hdfs-site.xml" (hdfsSiteDefault // hdfsSite // hdfsSiteInternal)}/* $out/ + cp ${siteXml "mapred-site.xml" (mapredSiteDefault // mapredSite)}/* $out/ + cp ${siteXml "yarn-site.xml" (yarnSiteDefault // yarnSite // yarnSiteInternal)}/* $out/ + cp ${siteXml "httpfs-site.xml" httpfsSite}/* $out/ + cp ${cfgFile "container-executor.cfg" containerExecutorCfg}/* $out/ + cp ${pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions}/* $out/ + cp ${pkgs.writeTextDir "hadoop-env.sh" hadoopEnv}/* $out/ + cp ${log4jProperties} $out/log4j.properties + ${lib.concatMapStringsSep "\n" (dir: "cp -r ${dir}/* $out/") extraConfDirs} +'') diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix new file mode 100644 index 00000000000..a4fdea81037 --- /dev/null +++ b/nixos/modules/services/cluster/hadoop/default.nix @@ -0,0 +1,223 @@ +{ config, lib, options, pkgs, ...}: +let + cfg = config.services.hadoop; + opt = options.services.hadoop; +in +with lib; +{ + imports = [ ./yarn.nix ./hdfs.nix ]; + + options.services.hadoop = { + coreSite = mkOption { + default = {}; + type = types.attrsOf types.anything; + example = literalExpression '' + { + "fs.defaultFS" = "hdfs://localhost"; + } + ''; + description = '' + Hadoop core-site.xml definition + <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/core-default.xml"/> + ''; + }; + coreSiteInternal = mkOption { + default = {}; + type = types.attrsOf types.anything; + internal = true; + description = '' + Internal option to add configs to core-site.xml based on module options + ''; + }; + + hdfsSiteDefault = mkOption { + default = { + "dfs.namenode.rpc-bind-host" = "0.0.0.0"; + "dfs.namenode.http-address" = "0.0.0.0:9870"; + "dfs.namenode.servicerpc-bind-host" = "0.0.0.0"; + "dfs.namenode.http-bind-host" = "0.0.0.0"; + }; + type = types.attrsOf types.anything; + description = '' + Default options for hdfs-site.xml + ''; + }; + hdfsSite = mkOption { + default = {}; + type = types.attrsOf types.anything; + example = literalExpression '' + { + "dfs.nameservices" = "namenode1"; + } + ''; + description = '' + Additional options and overrides for hdfs-site.xml + <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml"/> + ''; + }; + hdfsSiteInternal = mkOption { + default = {}; + type = types.attrsOf types.anything; + internal = true; + description = '' + Internal option to add configs to hdfs-site.xml based on module options + ''; + }; + + mapredSiteDefault = mkOption { + default = { + "mapreduce.framework.name" = "yarn"; + "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}"; + "mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}"; + "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}"; + }; + defaultText = literalExpression '' + { + "mapreduce.framework.name" = "yarn"; + "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}"; + "mapreduce.map.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}"; + "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}"; + } + ''; + type = types.attrsOf types.anything; + description = '' + Default options for mapred-site.xml + ''; + }; + mapredSite = mkOption { + default = {}; + type = types.attrsOf types.anything; + example = literalExpression '' + { + "mapreduce.map.java.opts" = "-Xmx900m -XX:+UseParallelGC"; + } + ''; + description = '' + Additional options and overrides for mapred-site.xml + <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml"/> + ''; + }; + + yarnSiteDefault = mkOption { + default = { + "yarn.nodemanager.admin-env" = "PATH=$PATH"; + "yarn.nodemanager.aux-services" = "mapreduce_shuffle"; + "yarn.nodemanager.aux-services.mapreduce_shuffle.class" = "org.apache.hadoop.mapred.ShuffleHandler"; + "yarn.nodemanager.bind-host" = "0.0.0.0"; + "yarn.nodemanager.container-executor.class" = "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor"; + "yarn.nodemanager.env-whitelist" = "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,LANG,TZ"; + "yarn.nodemanager.linux-container-executor.group" = "hadoop"; + "yarn.nodemanager.linux-container-executor.path" = "/run/wrappers/yarn-nodemanager/bin/container-executor"; + "yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager"; + "yarn.resourcemanager.bind-host" = "0.0.0.0"; + "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"; + }; + type = types.attrsOf types.anything; + description = '' + Default options for yarn-site.xml + ''; + }; + yarnSite = mkOption { + default = {}; + type = types.attrsOf types.anything; + example = literalExpression '' + { + "yarn.resourcemanager.hostname" = "''${config.networking.hostName}"; + } + ''; + description = '' + Additional options and overrides for yarn-site.xml + <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-common/yarn-default.xml"/> + ''; + }; + yarnSiteInternal = mkOption { + default = {}; + type = types.attrsOf types.anything; + internal = true; + description = '' + Internal option to add configs to yarn-site.xml based on module options + ''; + }; + + httpfsSite = mkOption { + default = { }; + type = types.attrsOf types.anything; + example = literalExpression '' + { + "hadoop.http.max.threads" = 500; + } + ''; + description = '' + Hadoop httpfs-site.xml definition + <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-hdfs-httpfs/httpfs-default.html"/> + ''; + }; + + log4jProperties = mkOption { + default = "${cfg.package}/lib/${cfg.package.untarDir}/etc/hadoop/log4j.properties"; + defaultText = literalExpression '' + "''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}/etc/hadoop/log4j.properties" + ''; + type = types.path; + example = literalExpression '' + "''${pkgs.hadoop}/lib/''${pkgs.hadoop.untarDir}/etc/hadoop/log4j.properties"; + ''; + description = "log4j.properties file added to HADOOP_CONF_DIR"; + }; + + containerExecutorCfg = mkOption { + default = { + # must be the same as yarn.nodemanager.linux-container-executor.group in yarnSite + "yarn.nodemanager.linux-container-executor.group"="hadoop"; + "min.user.id"=1000; + "feature.terminal.enabled"=1; + "feature.mount-cgroup.enabled" = 1; + }; + type = types.attrsOf types.anything; + example = literalExpression '' + options.services.hadoop.containerExecutorCfg.default // { + "feature.terminal.enabled" = 0; + } + ''; + description = '' + Yarn container-executor.cfg definition + <link xlink:href="https://hadoop.apache.org/docs/r2.7.2/hadoop-yarn/hadoop-yarn-site/SecureContainer.html"/> + ''; + }; + + extraConfDirs = mkOption { + default = []; + type = types.listOf types.path; + example = literalExpression '' + [ + ./extraHDFSConfs + ./extraYARNConfs + ] + ''; + description = "Directories containing additional config files to be added to HADOOP_CONF_DIR"; + }; + + gatewayRole.enable = mkEnableOption "gateway role for deploying hadoop configs"; + + package = mkOption { + type = types.package; + default = pkgs.hadoop; + defaultText = literalExpression "pkgs.hadoop"; + description = ""; + }; + }; + + + config = mkIf cfg.gatewayRole.enable { + users.groups.hadoop = { + gid = config.ids.gids.hadoop; + }; + environment = { + systemPackages = [ cfg.package ]; + etc."hadoop-conf".source = let + hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/"; + in "${hadoopConf}"; + variables.HADOOP_CONF_DIR = "/etc/hadoop-conf/"; + }; + }; +} diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix new file mode 100644 index 00000000000..325a002ad32 --- /dev/null +++ b/nixos/modules/services/cluster/hadoop/hdfs.nix @@ -0,0 +1,204 @@ +{ config, lib, pkgs, ... }: +with lib; +let + cfg = config.services.hadoop; + + # Config files for hadoop services + hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/"; + + # Generator for HDFS service options + hadoopServiceOption = { serviceName, firewallOption ? true, extraOpts ? null }: { + enable = mkEnableOption serviceName; + restartIfChanged = mkOption { + type = types.bool; + description = '' + Automatically restart the service on config change. + This can be set to false to defer restarts on clusters running critical applications. + Please consider the security implications of inadvertently running an older version, + and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option. + ''; + default = false; + }; + extraFlags = mkOption{ + type = with types; listOf str; + default = []; + description = "Extra command line flags to pass to ${serviceName}"; + example = [ + "-Dcom.sun.management.jmxremote" + "-Dcom.sun.management.jmxremote.port=8010" + ]; + }; + extraEnv = mkOption{ + type = with types; attrsOf str; + default = {}; + description = "Extra environment variables for ${serviceName}"; + }; + } // (optionalAttrs firewallOption { + openFirewall = mkOption { + type = types.bool; + default = false; + description = "Open firewall ports for ${serviceName}."; + }; + }) // (optionalAttrs (extraOpts != null) extraOpts); + + # Generator for HDFS service configs + hadoopServiceConfig = + { name + , serviceOptions ? cfg.hdfs."${toLower name}" + , description ? "Hadoop HDFS ${name}" + , User ? "hdfs" + , allowedTCPPorts ? [ ] + , preStart ? "" + , environment ? { } + , extraConfig ? { } + }: ( + + mkIf serviceOptions.enable ( mkMerge [{ + systemd.services."hdfs-${toLower name}" = { + inherit description preStart; + environment = environment // serviceOptions.extraEnv; + wantedBy = [ "multi-user.target" ]; + inherit (serviceOptions) restartIfChanged; + serviceConfig = { + inherit User; + SyslogIdentifier = "hdfs-${toLower name}"; + ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} ${toLower name} ${escapeShellArgs serviceOptions.extraFlags}"; + Restart = "always"; + }; + }; + + services.hadoop.gatewayRole.enable = true; + + networking.firewall.allowedTCPPorts = mkIf + ((builtins.hasAttr "openFirewall" serviceOptions) && serviceOptions.openFirewall) + allowedTCPPorts; + } extraConfig]) + ); + +in +{ + options.services.hadoop.hdfs = { + + namenode = hadoopServiceOption { serviceName = "HDFS NameNode"; } // { + formatOnInit = mkOption { + type = types.bool; + default = false; + description = '' + Format HDFS namenode on first start. This is useful for quickly spinning up + ephemeral HDFS clusters with a single namenode. + For HA clusters, initialization involves multiple steps across multiple nodes. + Follow this guide to initialize an HA cluster manually: + <link xlink:href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithQJM.html"/> + ''; + }; + }; + + datanode = hadoopServiceOption { serviceName = "HDFS DataNode"; } // { + dataDirs = mkOption { + default = null; + description = "Tier and path definitions for datanode storage."; + type = with types; nullOr (listOf (submodule { + options = { + type = mkOption { + type = enum [ "SSD" "DISK" "ARCHIVE" "RAM_DISK" ]; + description = '' + Storage types ([SSD]/[DISK]/[ARCHIVE]/[RAM_DISK]) for HDFS storage policies. + ''; + }; + path = mkOption { + type = path; + example = [ "/var/lib/hadoop/hdfs/dn" ]; + description = "Determines where on the local filesystem a data node should store its blocks."; + }; + }; + })); + }; + }; + + journalnode = hadoopServiceOption { serviceName = "HDFS JournalNode"; }; + + zkfc = hadoopServiceOption { + serviceName = "HDFS ZooKeeper failover controller"; + firewallOption = false; + }; + + httpfs = hadoopServiceOption { serviceName = "HDFS JournalNode"; } // { + tempPath = mkOption { + type = types.path; + default = "/tmp/hadoop/httpfs"; + description = "HTTPFS_TEMP path used by HTTPFS"; + }; + }; + + }; + + config = mkMerge [ + (hadoopServiceConfig { + name = "NameNode"; + allowedTCPPorts = [ + 9870 # namenode.http-address + 8020 # namenode.rpc-address + 8022 # namenode.servicerpc-address + 8019 # dfs.ha.zkfc.port + ]; + preStart = (mkIf cfg.hdfs.namenode.formatOnInit + "${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true" + ); + }) + + (hadoopServiceConfig { + name = "DataNode"; + # port numbers for datanode changed between hadoop 2 and 3 + allowedTCPPorts = if versionAtLeast cfg.package.version "3" then [ + 9864 # datanode.http.address + 9866 # datanode.address + 9867 # datanode.ipc.address + ] else [ + 50075 # datanode.http.address + 50010 # datanode.address + 50020 # datanode.ipc.address + ]; + extraConfig.services.hadoop.hdfsSiteInternal."dfs.datanode.data.dir" = let d = cfg.hdfs.datanode.dataDirs; in + if (d!= null) then (concatMapStringsSep "," (x: "["+x.type+"]file://"+x.path) cfg.hdfs.datanode.dataDirs) else d; + }) + + (hadoopServiceConfig { + name = "JournalNode"; + allowedTCPPorts = [ + 8480 # dfs.journalnode.http-address + 8485 # dfs.journalnode.rpc-address + ]; + }) + + (hadoopServiceConfig { + name = "zkfc"; + description = "Hadoop HDFS ZooKeeper failover controller"; + }) + + (hadoopServiceConfig { + name = "HTTPFS"; + environment.HTTPFS_TEMP = cfg.hdfs.httpfs.tempPath; + preStart = "mkdir -p $HTTPFS_TEMP"; + User = "httpfs"; + allowedTCPPorts = [ + 14000 # httpfs.http.port + ]; + }) + + (mkIf cfg.gatewayRole.enable { + users.users.hdfs = { + description = "Hadoop HDFS user"; + group = "hadoop"; + uid = config.ids.uids.hdfs; + }; + }) + (mkIf cfg.hdfs.httpfs.enable { + users.users.httpfs = { + description = "Hadoop HTTPFS user"; + group = "hadoop"; + isSystemUser = true; + }; + }) + + ]; +} diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix new file mode 100644 index 00000000000..74e16bdec68 --- /dev/null +++ b/nixos/modules/services/cluster/hadoop/yarn.nix @@ -0,0 +1,200 @@ +{ config, lib, pkgs, ...}: +with lib; +let + cfg = config.services.hadoop; + hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/"; + restartIfChanged = mkOption { + type = types.bool; + description = '' + Automatically restart the service on config change. + This can be set to false to defer restarts on clusters running critical applications. + Please consider the security implications of inadvertently running an older version, + and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option. + ''; + default = false; + }; + extraFlags = mkOption{ + type = with types; listOf str; + default = []; + description = "Extra command line flags to pass to the service"; + example = [ + "-Dcom.sun.management.jmxremote" + "-Dcom.sun.management.jmxremote.port=8010" + ]; + }; + extraEnv = mkOption{ + type = with types; attrsOf str; + default = {}; + description = "Extra environment variables"; + }; +in +{ + options.services.hadoop.yarn = { + resourcemanager = { + enable = mkEnableOption "Hadoop YARN ResourceManager"; + inherit restartIfChanged extraFlags extraEnv; + + openFirewall = mkOption { + type = types.bool; + default = false; + description = '' + Open firewall ports for resourcemanager + ''; + }; + }; + nodemanager = { + enable = mkEnableOption "Hadoop YARN NodeManager"; + inherit restartIfChanged extraFlags extraEnv; + + resource = { + cpuVCores = mkOption { + description = "Number of vcores that can be allocated for containers."; + type = with types; nullOr ints.positive; + default = null; + }; + maximumAllocationVCores = mkOption { + description = "The maximum virtual CPU cores any container can be allocated."; + type = with types; nullOr ints.positive; + default = null; + }; + memoryMB = mkOption { + description = "Amount of physical memory, in MB, that can be allocated for containers."; + type = with types; nullOr ints.positive; + default = null; + }; + maximumAllocationMB = mkOption { + description = "The maximum physical memory any container can be allocated."; + type = with types; nullOr ints.positive; + default = null; + }; + }; + + useCGroups = mkOption { + type = types.bool; + default = true; + description = '' + Use cgroups to enforce resource limits on containers + ''; + }; + + localDir = mkOption { + description = "List of directories to store localized files in."; + type = with types; nullOr (listOf path); + example = [ "/var/lib/hadoop/yarn/nm" ]; + default = null; + }; + + addBinBash = mkOption { + type = types.bool; + default = true; + description = '' + Add /bin/bash. This is needed by the linux container executor's launch script. + ''; + }; + openFirewall = mkOption { + type = types.bool; + default = false; + description = '' + Open firewall ports for nodemanager. + Because containers can listen on any ephemeral port, TCP ports 1024–65535 will be opened. + ''; + }; + }; + }; + + config = mkMerge [ + (mkIf cfg.gatewayRole.enable { + users.users.yarn = { + description = "Hadoop YARN user"; + group = "hadoop"; + uid = config.ids.uids.yarn; + }; + }) + + (mkIf cfg.yarn.resourcemanager.enable { + systemd.services.yarn-resourcemanager = { + description = "Hadoop YARN ResourceManager"; + wantedBy = [ "multi-user.target" ]; + inherit (cfg.yarn.resourcemanager) restartIfChanged; + environment = cfg.yarn.resourcemanager.extraEnv; + + serviceConfig = { + User = "yarn"; + SyslogIdentifier = "yarn-resourcemanager"; + ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " + + " resourcemanager ${escapeShellArgs cfg.yarn.resourcemanager.extraFlags}"; + Restart = "always"; + }; + }; + + services.hadoop.gatewayRole.enable = true; + + networking.firewall.allowedTCPPorts = (mkIf cfg.yarn.resourcemanager.openFirewall [ + 8088 # resourcemanager.webapp.address + 8030 # resourcemanager.scheduler.address + 8031 # resourcemanager.resource-tracker.address + 8032 # resourcemanager.address + 8033 # resourcemanager.admin.address + ]); + }) + + (mkIf cfg.yarn.nodemanager.enable { + # Needed because yarn hardcodes /bin/bash in container start scripts + # These scripts can't be patched, they are generated at runtime + systemd.tmpfiles.rules = [ + (mkIf cfg.yarn.nodemanager.addBinBash "L /bin/bash - - - - /run/current-system/sw/bin/bash") + ]; + + systemd.services.yarn-nodemanager = { + description = "Hadoop YARN NodeManager"; + wantedBy = [ "multi-user.target" ]; + inherit (cfg.yarn.nodemanager) restartIfChanged; + environment = cfg.yarn.nodemanager.extraEnv; + + preStart = '' + # create log dir + mkdir -p /var/log/hadoop/yarn/nodemanager + chown yarn:hadoop /var/log/hadoop/yarn/nodemanager + + # set up setuid container executor binary + umount /run/wrappers/yarn-nodemanager/cgroup/cpu || true + rm -rf /run/wrappers/yarn-nodemanager/ || true + mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop,cgroup/cpu} + cp ${cfg.package}/lib/${cfg.package.untarDir}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/ + chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor + chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor + cp ${hadoopConf}/container-executor.cfg /run/wrappers/yarn-nodemanager/etc/hadoop/ + ''; + + serviceConfig = { + User = "yarn"; + SyslogIdentifier = "yarn-nodemanager"; + PermissionsStartOnly = true; + ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " + + " nodemanager ${escapeShellArgs cfg.yarn.nodemanager.extraFlags}"; + Restart = "always"; + }; + }; + + services.hadoop.gatewayRole.enable = true; + + services.hadoop.yarnSiteInternal = with cfg.yarn.nodemanager; { + "yarn.nodemanager.local-dirs" = localDir; + "yarn.scheduler.maximum-allocation-vcores" = resource.maximumAllocationVCores; + "yarn.scheduler.maximum-allocation-mb" = resource.maximumAllocationMB; + "yarn.nodemanager.resource.cpu-vcores" = resource.cpuVCores; + "yarn.nodemanager.resource.memory-mb" = resource.memoryMB; + } // mkIf useCGroups { + "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn"; + "yarn.nodemanager.linux-container-executor.resources-handler.class" = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler"; + "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true"; + "yarn.nodemanager.linux-container-executor.cgroups.mount-path" = "/run/wrappers/yarn-nodemanager/cgroup"; + }; + + networking.firewall.allowedTCPPortRanges = [ + (mkIf (cfg.yarn.nodemanager.openFirewall) {from = 1024; to = 65535;}) + ]; + }) + + ]; +} |