Merge pull request #21416 from cstrahan/mesos-1.1.0

mesos: 1.0.1 -> 1.1.0

4 files changed, 265 insertions, 23 deletions

diff --git a/nixos/modules/services/misc/mesos-master.nix b/nixos/modules/services/misc/mesos-master.nix
index 99583ebeebd..0523c6549ed 100644
--- a/nixos/modules/services/misc/mesos-master.nix
+++ b/nixos/modules/services/misc/mesos-master.nix
@@ -16,12 +16,30 @@ in {
         type = types.bool;
       };
 
+      ip = mkOption {
+        description = "IP address to listen on.";
+        default = "0.0.0.0";
+        type = types.str;
+      };
+
       port = mkOption {
         description = "Mesos Master port";
         default = 5050;
         type = types.int;
       };
 
+      advertiseIp = mkOption {
+        description = "IP address advertised to reach this master.";
+        default = null;
+        type = types.nullOr types.str;
+      };
+
+      advertisePort = mkOption {
+        description = "Port advertised to reach this Mesos master.";
+        default = null;
+        type = types.nullOr types.int;
+      };
+
       zk = mkOption {
         description = ''
           ZooKeeper URL (used for leader election amongst masters).
@@ -84,7 +102,10 @@ in {
       serviceConfig = {
         ExecStart = ''
           ${pkgs.mesos}/bin/mesos-master \
+            --ip=${cfg.ip} \
             --port=${toString cfg.port} \
+            ${optionalString (cfg.advertiseIp != null) "--advertise_ip=${cfg.advertiseIp}"} \
+            ${optionalString (cfg.advertisePort  != null) "--advertise_port=${toString cfg.advertisePort}"} \
             ${if cfg.quorum == 0
               then "--registry=in_memory"
               else "--zk=${cfg.zk} --registry=replicated_log --quorum=${toString cfg.quorum}"} \
diff --git a/nixos/modules/services/misc/mesos-slave.nix b/nixos/modules/services/misc/mesos-slave.nix
index 9ddecb6fe30..47be10274d3 100644
--- a/nixos/modules/services/misc/mesos-slave.nix
+++ b/nixos/modules/services/misc/mesos-slave.nix
@@ -12,7 +12,23 @@ let
   attribsArg = optionalString (cfg.attributes != {})
                               "--attributes=${mkAttributes cfg.attributes}";
 
-  containerizers = [ "mesos" ] ++ (optional cfg.withDocker "docker");
+  containerizersArg = concatStringsSep "," (
+    lib.unique (
+      cfg.containerizers ++ (optional cfg.withDocker "docker")
+    )
+  );
+
+  imageProvidersArg = concatStringsSep "," (
+    lib.unique (
+      cfg.imageProviders ++ (optional cfg.withDocker "docker")
+    )
+  );
+
+  isolationArg = concatStringsSep "," (
+    lib.unique (
+      cfg.isolation ++ (optionals cfg.withDocker [ "filesystem/linux" "docker/runtime"])
+    )
+  );
 
 in {
 
@@ -27,7 +43,7 @@ in {
       ip = mkOption {
         description = "IP address to listen on.";
         default = "0.0.0.0";
-        type = types.string;
+        type = types.str;
       };
 
       port = mkOption {
@@ -36,6 +52,53 @@ in {
         type = types.int;
       };
 
+      advertiseIp = mkOption {
+        description = "IP address advertised to reach this agent.";
+        default = null;
+        type = types.nullOr types.str;
+      };
+
+      advertisePort = mkOption {
+        description = "Port advertised to reach this agent.";
+        default = null;
+        type = types.nullOr types.int;
+      };
+
+      containerizers = mkOption {
+        description = ''
+          List of containerizer implementations to compose in order to provide
+          containerization. Available options are mesos and docker.
+          The order the containerizers are specified is the order they are tried.
+        '';
+        default = [ "mesos" ];
+        type = types.listOf types.str;
+      };
+
+      imageProviders = mkOption {
+        description = "List of supported image providers, e.g., APPC,DOCKER.";
+        default = [ ];
+        type = types.listOf types.str;
+      };
+
+      imageProvisionerBackend = mkOption {
+        description = ''
+          Strategy for provisioning container rootfs from images,
+          e.g., aufs, bind, copy, overlay.
+        '';
+        default = "copy";
+        type = types.str;
+      };
+
+      isolation = mkOption {
+        description = ''
+          Isolation mechanisms to use, e.g., posix/cpu,posix/mem, or
+          cgroups/cpu,cgroups/mem, or network/port_mapping, or `gpu/nvidia` for nvidia
+          specific gpu isolation.
+        '';
+        default = [ "posix/cpu" "posix/mem" ];
+        type = types.listOf types.str;
+      };
+
       master = mkOption {
         description = ''
           May be one of:
@@ -57,6 +120,16 @@ in {
         type = types.bool;
       };
 
+      dockerRegistry = mkOption {
+        description = ''
+          The default url for pulling Docker images.
+          It could either be a Docker registry server url,
+          or a local path in which Docker image archives are stored.
+        '';
+        default = null;
+        type = types.nullOr (types.either types.str types.path);
+      };
+
       workDir = mkOption {
         description = "The Mesos work directory.";
         default = "/var/lib/mesos/slave";
@@ -96,28 +169,45 @@ in {
                     host = "aabc123";
                     os = "nixos"; };
       };
+
+      executorEnvironmentVariables = mkOption {
+        description = ''
+          The environment variables that should be passed to the executor, and thus subsequently task(s).
+        '';
+        default = {
+          PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
+        };
+        type = types.attrsOf types.str;
+      };
     };
 
   };
 
-
   config = mkIf cfg.enable {
     systemd.services.mesos-slave = {
       description = "Mesos Slave";
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];
-      environment.MESOS_CONTAINERIZERS = concatStringsSep "," containerizers;
+      path = [ pkgs.stdenv.shellPackage ];
       serviceConfig = {
         ExecStart = ''
           ${pkgs.mesos}/bin/mesos-slave \
+            --containerizers=${containerizersArg} \
+            --image_providers=${imageProvidersArg} \
+            --image_provisioner_backend=${cfg.imageProvisionerBackend} \
+            --isolation=${isolationArg} \
             --ip=${cfg.ip} \
             --port=${toString cfg.port} \
+            ${optionalString (cfg.advertiseIp != null) "--advertise_ip=${cfg.advertiseIp}"} \
+            ${optionalString (cfg.advertisePort  != null) "--advertise_port=${toString cfg.advertisePort}"} \
             --master=${cfg.master} \
             --work_dir=${cfg.workDir} \
             --logging_level=${cfg.logLevel} \
             ${attribsArg} \
             ${optionalString cfg.withHadoop "--hadoop-home=${pkgs.hadoop}"} \
             ${optionalString cfg.withDocker "--docker=${pkgs.docker}/libexec/docker/docker"} \
+            ${optionalString (cfg.dockerRegistry != null) "--docker_registry=${cfg.dockerRegistry}"} \
+            --executor_environment_variables=${lib.escapeShellArg (builtins.toJSON cfg.executorEnvironmentVariables)} \
             ${toString cfg.extraCmdLineOptions}
         '';
         PermissionsStartOnly = true;
diff --git a/nixos/tests/mesos.nix b/nixos/tests/mesos.nix
index 3610603aeba..6e9af126f03 100644
--- a/nixos/tests/mesos.nix
+++ b/nixos/tests/mesos.nix
@@ -1,32 +1,91 @@
-import ./make-test.nix ({ pkgs, ...} : {
-  name = "simple";
+import ./make-test.nix ({ pkgs, ...} : rec {
+  name = "mesos";
   meta = with pkgs.stdenv.lib.maintainers; {
-    maintainers = [ offline ];
+    maintainers = [ offline kamilchm cstrahan ];
   };
 
-  machine = { config, pkgs, ... }: {
-    services.zookeeper.enable = true;
-    virtualisation.docker.enable = true;
-    services.mesos = {
-      slave = {
-        enable = true;
-        master = "zk://localhost:2181/mesos";
-        attributes = {
-          tag1 = "foo";
-          tag2 = "bar";
-        };
+  nodes = {
+    master = { config, pkgs, ... }: {
+      networking.firewall.enable = false;
+      services.zookeeper.enable = true;
+      services.mesos.master = {
+          enable = true;
+          zk = "zk://master:2181/mesos";
       };
-      master = {
-        enable = true;
-        zk = "zk://localhost:2181/mesos";
+    };
+
+    slave = { config, pkgs, ... }: {
+      networking.firewall.enable = false;
+      networking.nat.enable = true;
+      virtualisation.docker.enable = true;
+      services.mesos = {
+        slave = {
+          enable = true;
+          master = "master:5050";
+          dockerRegistry = registry;
+          executorEnvironmentVariables = {
+            PATH = "/run/current-system/sw/bin";
+          };
+        };
       };
     };
   };
 
+  simpleDocker = pkgs.dockerTools.buildImage {
+    name = "echo";
+    contents = [ pkgs.stdenv.shellPackage pkgs.coreutils ];
+    config = {
+      Env = [
+        # When shell=true, mesos invokes "sh -c '<cmd>'", so make sure "sh" is
+        # on the PATH.
+        "PATH=${pkgs.stdenv.shellPackage}/bin:${pkgs.coreutils}/bin"
+      ];
+      Entrypoint = [ "echo" ];
+    };
+  };
+
+  registry = pkgs.runCommand "registry" { } ''
+    mkdir -p $out
+    cp ${simpleDocker} $out/echo:latest.tar
+  '';
+
+  testFramework = pkgs.pythonPackages.buildPythonPackage {
+    name = "mesos-tests";
+    propagatedBuildInputs = [ pkgs.mesos ];
+    catchConflicts = false;
+    src = ./mesos_test.py;
+    phases = [ "installPhase" "fixupPhase" ];
+    installPhase = ''
+      mkdir $out
+      cp $src $out/mesos_test.py
+      chmod +x $out/mesos_test.py
+
+      echo "done" > test.result
+      tar czf $out/test.tar.gz test.result
+    '';
+  };
+
   testScript =
     ''
       startAll;
-      $machine->waitForUnit("mesos-master.service");
-      $machine->waitForUnit("mesos-slave.service");
+      $master->waitForUnit("mesos-master.service");
+      $slave->waitForUnit("mesos-slave.service");
+
+      $master->waitForOpenPort(5050);
+      $slave->waitForOpenPort(5051);
+
+      # is slave registred? 
+      $master->waitUntilSucceeds("curl -s --fail http://master:5050/master/slaves".
+                                 " | grep -q \"\\\"hostname\\\":\\\"slave\\\"\"");
+
+      # try to run docker image 
+      $master->succeed("${pkgs.mesos}/bin/mesos-execute --master=master:5050".
+                       " --resources=\"cpus:0.1;mem:32\" --name=simple-docker".
+                       " --containerizer=mesos --docker_image=echo:latest".
+                       " --shell=true --command=\"echo done\" | grep -q TASK_FINISHED");
+
+      # simple command with .tar.gz uri
+      $master->succeed("${testFramework}/mesos_test.py master ".
+                       "${testFramework}/test.tar.gz");
     '';
 })
diff --git a/nixos/tests/mesos_test.py b/nixos/tests/mesos_test.py
new file mode 100644
index 00000000000..be8bb32e49a
--- /dev/null
+++ b/nixos/tests/mesos_test.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+import uuid
+import time
+import subprocess
+import os
+
+import sys
+
+from mesos.interface import Scheduler
+from mesos.native import MesosSchedulerDriver
+from mesos.interface import mesos_pb2
+
+def log(msg):
+    process = subprocess.Popen("systemd-cat", stdin=subprocess.PIPE)
+    (out,err) = process.communicate(msg)
+
+class NixosTestScheduler(Scheduler):
+    def __init__(self):
+        self.master_ip = sys.argv[1]
+        self.download_uri = sys.argv[2]
+
+    def resourceOffers(self, driver, offers):
+        log("XXX got resource offer")
+
+        offer = offers[0]
+        task = self.new_task(offer)
+        uri = task.command.uris.add()
+        uri.value = self.download_uri
+        task.command.value = "cat test.result"
+        driver.launchTasks(offer.id, [task])
+
+    def statusUpdate(self, driver, update):
+        log("XXX status update")
+        if update.state == mesos_pb2.TASK_FAILED:
+            log("XXX test task failed with message: " + update.message)
+            driver.stop()
+            sys.exit(1)
+        elif update.state == mesos_pb2.TASK_FINISHED:
+            driver.stop()
+            sys.exit(0)
+
+    def new_task(self, offer):
+        task = mesos_pb2.TaskInfo()
+        id = uuid.uuid4()
+        task.task_id.value = str(id)
+        task.slave_id.value = offer.slave_id.value
+        task.name = "task {}".format(str(id))
+
+        cpus = task.resources.add()
+        cpus.name = "cpus"
+        cpus.type = mesos_pb2.Value.SCALAR
+        cpus.scalar.value = 0.1
+
+        mem = task.resources.add()
+        mem.name = "mem"
+        mem.type = mesos_pb2.Value.SCALAR
+        mem.scalar.value = 32
+
+        return task
+
+if __name__ == '__main__':
+    log("XXX framework started")
+
+    framework = mesos_pb2.FrameworkInfo()
+    framework.user = "root"
+    framework.name = "nixos-test-framework"
+    driver = MesosSchedulerDriver(
+        NixosTestScheduler(),
+        framework,
+        sys.argv[1] + ":5050"
+    )
+    driver.run()