diff options
| author | illustris <rharikrishnan95@gmail.com> | 2022-02-28 20:41:43 +0530 |
|---|---|---|
| committer | illustris <rharikrishnan95@gmail.com> | 2022-03-11 14:18:44 +0530 |
| commit | bef71d7c530aeecdcfb66290e6c0b4948d1fb223 (patch) | |
| tree | e695e5fe5b2deb63613a2a9868714cfa30cefa5f | |
| parent | 716b0dfaaf12afe83ff54b793dc52c022ab62155 (diff) | |
| download | nixpkgs-bef71d7c530aeecdcfb66290e6c0b4948d1fb223.tar nixpkgs-bef71d7c530aeecdcfb66290e6c0b4948d1fb223.tar.gz nixpkgs-bef71d7c530aeecdcfb66290e6c0b4948d1fb223.tar.bz2 nixpkgs-bef71d7c530aeecdcfb66290e6c0b4948d1fb223.tar.lz nixpkgs-bef71d7c530aeecdcfb66290e6c0b4948d1fb223.tar.xz nixpkgs-bef71d7c530aeecdcfb66290e6c0b4948d1fb223.tar.zst nixpkgs-bef71d7c530aeecdcfb66290e6c0b4948d1fb223.zip | |
nixos/hadoop: use CGroups to enforce container limits by default
| -rw-r--r-- | nixos/modules/services/cluster/hadoop/default.nix | 5 | ||||
| -rw-r--r-- | nixos/modules/services/cluster/hadoop/yarn.nix | 3 | ||||
| -rw-r--r-- | nixos/tests/hadoop/yarn.nix | 4 |
3 files changed, 9 insertions, 3 deletions
diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix index 5c7ea79c301..57b1d7a90d7 100644 --- a/nixos/modules/services/cluster/hadoop/default.nix +++ b/nixos/modules/services/cluster/hadoop/default.nix @@ -79,6 +79,10 @@ with lib; "yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager"; "yarn.resourcemanager.bind-host" = "0.0.0.0"; "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"; + "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn"; + "yarn.nodemanager.linux-container-executor.resources-handler.class" = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler"; + "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true"; + "yarn.nodemanager.linux-container-executor.cgroups.mount-path" = "/run/wrappers/yarn-nodemanager/cgroup"; }; type = types.attrsOf types.anything; example = literalExpression '' @@ -124,6 +128,7 @@ with lib; "yarn.nodemanager.linux-container-executor.group"="hadoop"; "min.user.id"=1000; "feature.terminal.enabled"=1; + "feature.mount-cgroup.enabled" = 1; }; type = types.attrsOf types.anything; example = literalExpression '' diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix index 9f0d3f85db7..373d8a70a12 100644 --- a/nixos/modules/services/cluster/hadoop/yarn.nix +++ b/nixos/modules/services/cluster/hadoop/yarn.nix @@ -101,8 +101,9 @@ in chown yarn:hadoop /var/log/hadoop/yarn/nodemanager # set up setuid container executor binary + umount /run/wrappers/yarn-nodemanager/cgroup/cpu || true rm -rf /run/wrappers/yarn-nodemanager/ || true - mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop} + mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop,cgroup/cpu} cp ${cfg.package}/lib/${cfg.package.untarDir}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/ chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor diff --git a/nixos/tests/hadoop/yarn.nix b/nixos/tests/hadoop/yarn.nix index ba39e257dc8..1bf8e3831f6 100644 --- a/nixos/tests/hadoop/yarn.nix +++ b/nixos/tests/hadoop/yarn.nix @@ -12,14 +12,14 @@ import ../make-test-python.nix ({ package, ... }: { }; }; }; - nodemanager = { ... }: { + nodemanager = { options, lib, ... }: { services.hadoop = { inherit package; yarn.nodemanager = { enable = true; openFirewall = true; }; - yarnSite = { + yarnSite = options.services.hadoop.yarnSite.default // { "yarn.resourcemanager.hostname" = "resourcemanager"; "yarn.nodemanager.log-dirs" = "/tmp/userlogs"; }; |