From 0c10b2baa6bf61c8ddaed7cdb6c2f2dbaab42662 Mon Sep 17 00:00:00 2001
From: Allan Espinosa <allan.espinosa@outlook.com>
Date: Sun, 20 May 2018 21:09:31 -0400
Subject: nixos/hadoop: add hadoop module (hdfs, yarn)

---
 nixos/tests/hadoop/hdfs.nix | 54 +++++++++++++++++++++++++++++++++++++++++++++
 nixos/tests/hadoop/yarn.nix | 46 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 nixos/tests/hadoop/hdfs.nix
 create mode 100644 nixos/tests/hadoop/yarn.nix

(limited to 'nixos/tests/hadoop')

diff --git a/nixos/tests/hadoop/hdfs.nix b/nixos/tests/hadoop/hdfs.nix
new file mode 100644
index 00000000000..4206c940c1a
--- /dev/null
+++ b/nixos/tests/hadoop/hdfs.nix
@@ -0,0 +1,54 @@
+import ../make-test.nix ({pkgs, ...}: {
+  nodes = {
+    namenode = {pkgs, config, ...}: {
+      services.hadoop = {
+        package = pkgs.hadoop_3_1;
+        hdfs.namenode.enabled = true;
+        coreSite = {
+          "fs.defaultFS" = "hdfs://namenode:8020";
+        };
+        hdfsSite = {
+          "dfs.replication" = 1;
+          "dfs.namenode.rpc-bind-host" = "0.0.0.0";
+          "dfs.namenode.http-bind-host" = "0.0.0.0";
+        };
+      };
+      networking.firewall.allowedTCPPorts = [
+        9870 # namenode.http-address
+        8020 # namenode.rpc-address
+      ];
+    };
+    datanode = {pkgs, config, ...}: {
+      services.hadoop = {
+        package = pkgs.hadoop_3_1;
+        hdfs.datanode.enabled = true;
+        coreSite = {
+          "fs.defaultFS" = "hdfs://namenode:8020";
+        };
+      };
+      networking.firewall.allowedTCPPorts = [
+        9864 # datanode.http.address
+        9866 # datanode.address
+        9867 # datanode.ipc.address
+      ];
+    };
+  };
+
+  testScript = ''
+    startAll
+
+    $namenode->waitForUnit("hdfs-namenode");
+    $namenode->waitForUnit("network.target");
+    $namenode->waitForOpenPort(8020);
+    $namenode->waitForOpenPort(9870);
+
+    $datanode->waitForUnit("hdfs-datanode");
+    $datanode->waitForUnit("network.target");
+    $datanode->waitForOpenPort(9864);
+    $datanode->waitForOpenPort(9866);
+    $datanode->waitForOpenPort(9867);
+
+    $namenode->succeed("curl http://namenode:9870");
+    $datanode->succeed("curl http://datanode:9864");
+  '';
+})
diff --git a/nixos/tests/hadoop/yarn.nix b/nixos/tests/hadoop/yarn.nix
new file mode 100644
index 00000000000..e97cc1acc90
--- /dev/null
+++ b/nixos/tests/hadoop/yarn.nix
@@ -0,0 +1,46 @@
+import ../make-test.nix ({pkgs, ...}: {
+  nodes = {
+    resourcemanager = {pkgs, config, ...}: {
+      services.hadoop.package = pkgs.hadoop_3_1;
+      services.hadoop.yarn.resourcemanager.enabled = true;
+      services.hadoop.yarnSite = {
+        "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
+      };
+      networking.firewall.allowedTCPPorts = [
+        8088 # resourcemanager.webapp.address
+        8031 # resourcemanager.resource-tracker.address
+      ];
+    };
+    nodemanager = {pkgs, config, ...}: {
+      services.hadoop.package = pkgs.hadoop_3_1;
+      services.hadoop.yarn.nodemanager.enabled = true;
+      services.hadoop.yarnSite = {
+        "yarn.resourcemanager.hostname" = "resourcemanager";
+        "yarn.nodemanager.log-dirs" = "/tmp/userlogs";
+        "yarn.nodemanager.address" = "0.0.0.0:8041";
+      };
+      networking.firewall.allowedTCPPorts = [
+        8042 # nodemanager.webapp.address
+        8041 # nodemanager.address
+      ];
+    };
+
+  };
+
+  testScript = ''
+    startAll;
+
+    $resourcemanager->waitForUnit("yarn-resourcemanager");
+    $resourcemanager->waitForUnit("network.target");
+    $resourcemanager->waitForOpenPort(8031);
+    $resourcemanager->waitForOpenPort(8088);
+
+    $nodemanager->waitForUnit("yarn-nodemanager");
+    $nodemanager->waitForUnit("network.target");
+    $nodemanager->waitForOpenPort(8042);
+    $nodemanager->waitForOpenPort(8041);
+
+    $resourcemanager->succeed("curl http://localhost:8088");
+    $nodemanager->succeed("curl http://localhost:8042");
+  '';
+})
-- 
cgit 1.4.1