diff options
| -rw-r--r-- | nixos/modules/module-list.nix | 2 | ||||
| -rw-r--r-- | nixos/modules/services/cluster/kubernetes/dashboard.nix | 85 | ||||
| -rw-r--r-- | nixos/modules/services/cluster/kubernetes/default.nix (renamed from nixos/modules/services/cluster/kubernetes.nix) | 281 | ||||
| -rw-r--r-- | nixos/modules/services/cluster/kubernetes/dns.nix | 240 | ||||
| -rw-r--r-- | nixos/modules/services/cluster/kubernetes/kube-addon-manager.nix | 54 |
5 files changed, 612 insertions, 50 deletions
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 7edea2c9538..cb71e93e578 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -157,7 +157,7 @@ ./services/backup/tarsnap.nix ./services/backup/znapzend.nix ./services/cluster/fleet.nix - ./services/cluster/kubernetes.nix + ./services/cluster/kubernetes/default.nix ./services/cluster/panamax.nix ./services/computing/boinc/client.nix ./services/computing/torque/server.nix diff --git a/nixos/modules/services/cluster/kubernetes/dashboard.nix b/nixos/modules/services/cluster/kubernetes/dashboard.nix new file mode 100644 index 00000000000..337c2634a37 --- /dev/null +++ b/nixos/modules/services/cluster/kubernetes/dashboard.nix @@ -0,0 +1,85 @@ +{ cfg }: { + "dashboard-controller" = { + "apiVersion" = "extensions/v1beta1"; + "kind" = "Deployment"; + "metadata" = { + "labels" = { + "addonmanager.kubernetes.io/mode" = "Reconcile"; + "k8s-app" = "kubernetes-dashboard"; + "kubernetes.io/cluster-service" = "true"; + }; + "name" = "kubernetes-dashboard"; + "namespace" = "kube-system"; + }; + "spec" = { + "selector" = { + "matchLabels" = { + "k8s-app" = "kubernetes-dashboard"; + }; + }; + "template" = { + "metadata" = { + "annotations" = { + "scheduler.alpha.kubernetes.io/critical-pod" = ""; + }; + "labels" = { + "k8s-app" = "kubernetes-dashboard"; + }; + }; + "spec" = { + "containers" = [{ + "image" = "gcr.io/google_containers/kubernetes-dashboard-amd64:v1.6.0"; + "livenessProbe" = { + "httpGet" = { + "path" = "/"; + "port" = 9090; + }; + "initialDelaySeconds" = 30; + "timeoutSeconds" = 30; + }; + "name" = "kubernetes-dashboard"; + "ports" = [{ + "containerPort" = 9090; + }]; + "resources" = { + "limits" = { + "cpu" = "100m"; + "memory" = "50Mi"; + }; + "requests" = { + "cpu" = "100m"; + "memory" = "50Mi"; + }; + }; + }]; + "tolerations" = [{ + "key" = "CriticalAddonsOnly"; + "operator" = "Exists"; + }]; + }; + }; + }; + }; + "dashboard-service" = { + "apiVersion" = "v1"; + "kind" = "Service"; + "metadata" = { + "labels" = { + "addonmanager.kubernetes.io/mode" = "Reconcile"; + "k8s-app" = "kubernetes-dashboard"; + "kubernetes.io/cluster-service" = "true"; + }; + "name" = "kubernetes-dashboard"; + "namespace" = "kube-system"; + }; + "spec" = { + "ports" = [{ + "port" = 80; + "targetPort" = 9090; + }]; + "selector" = { + "k8s-app" = "kubernetes-dashboard"; + }; + }; + }; +} diff --git a/nixos/modules/services/cluster/kubernetes.nix b/nixos/modules/services/cluster/kubernetes/default.nix index 8177233b3ed..7160bcca153 100644 --- a/nixos/modules/services/cluster/kubernetes.nix +++ b/nixos/modules/services/cluster/kubernetes/default.nix @@ -53,6 +53,35 @@ let ) cfg.kubelet.manifests; }; + addons = pkgs.runCommand "kubernetes-addons" { } '' + mkdir -p $out + # since we are mounting the addons to the addon manager, they need to be copied + ${concatMapStringsSep ";" (a: "cp -v ${a}/* $out/") (mapAttrsToList (name: addon: + pkgs.writeTextDir "${name}.json" (builtins.toJSON addon) + ) (cfg.addonManager.addons))} + ''; + + taintOptions = { name, ... }: { + options = { + key = mkOption { + description = "Key of taint."; + default = name; + type = types.str; + }; + value = mkOption { + description = "Value of taint."; + type = types.str; + }; + effect = mkOption { + description = "Effect of taint."; + example = "NoSchedule"; + type = types.str; + }; + }; + }; + + taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.kubelet.taints); + in { ###### interface @@ -77,7 +106,7 @@ in { }; verbose = mkOption { - description = "Kubernetes enable verbose mode for debugging"; + description = "Kubernetes enable verbose mode for debugging."; default = false; type = types.bool; }; @@ -90,19 +119,19 @@ in { }; keyFile = mkOption { - description = "Etcd key file"; + description = "Etcd key file."; default = null; type = types.nullOr types.path; }; certFile = mkOption { - description = "Etcd cert file"; + description = "Etcd cert file."; default = null; type = types.nullOr types.path; }; caFile = mkOption { - description = "Etcd ca file"; + description = "Etcd ca file."; default = null; type = types.nullOr types.path; }; @@ -110,25 +139,25 @@ in { kubeconfig = { server = mkOption { - description = "Kubernetes apiserver server address"; + description = "Kubernetes apiserver server address."; default = "http://${cfg.apiserver.address}:${toString cfg.apiserver.port}"; type = types.str; }; caFile = mkOption { - description = "Certificate authrority file to use to connect to kuberentes apiserver"; + description = "Certificate authrority file to use to connect to Kubernetes apiserver."; type = types.nullOr types.path; default = null; }; certFile = mkOption { - description = "Client certificate file to use to connect to kubernetes"; + description = "Client certificate file to use to connect to Kubernetes."; type = types.nullOr types.path; default = null; }; keyFile = mkOption { - description = "Client key file to use to connect to kubernetes"; + description = "Client key file to use to connect to Kubernetes."; type = types.nullOr types.path; default = null; }; @@ -142,7 +171,7 @@ in { apiserver = { enable = mkOption { - description = "Whether to enable kubernetes apiserver."; + description = "Whether to enable Kubernetes apiserver."; default = false; type = types.bool; }; @@ -172,6 +201,14 @@ in { type = types.nullOr types.str; }; + storageBackend = mkOption { + description = '' + Kubernetes apiserver storage backend. + ''; + default = "etcd2"; + type = types.enum ["etcd2" "etcd3"]; + }; + port = mkOption { description = "Kubernetes apiserver listening port."; default = 8080; @@ -227,7 +264,7 @@ in { Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/RBAC). See <link xlink:href="http://kubernetes.io/docs/admin/authorization.html"/> ''; - default = ["ABAC" "RBAC"]; + default = ["ABAC"]; type = types.listOf (types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "RBAC"]); }; @@ -274,7 +311,29 @@ in { apiVersion = "abac.authorization.kubernetes.io/v1beta1"; kind = "Policy"; spec = { - user = "kube"; + user = "kube-worker"; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + user = "kube_proxy"; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + user = "client"; namespace = "*"; resource = "*"; apiGroup = "*"; @@ -285,7 +344,7 @@ in { apiVersion = "abac.authorization.kubernetes.io/v1beta1"; kind = "Policy"; spec = { - user = "system:serviceaccount:kube-system:default"; + group = "system:serviceaccounts"; namespace = "*"; resource = "*"; apiGroup = "*"; @@ -297,19 +356,19 @@ in { }; autorizationRBACSuperAdmin = mkOption { - description = "Role based authorization super admin"; + description = "Role based authorization super admin."; default = "admin"; type = types.str; }; allowPrivileged = mkOption { - description = "Whether to allow privileged containers on kubernetes."; + description = "Whether to allow privileged containers on Kubernetes."; default = true; type = types.bool; }; portalNet = mkOption { - description = "Kubernetes CIDR notation IP range from which to assign portal IPs"; + description = "Kubernetes CIDR notation IP range from which to assign portal IPs."; default = "10.10.10.10/24"; type = types.str; }; @@ -319,7 +378,7 @@ in { Api runtime configuration. See <link xlink:href="http://kubernetes.io/v1.0/docs/admin/cluster-management.html"/> ''; - default = "rbac.authorization.k8s.io/v1alpha1"; + default = ""; example = "api/all=false,api/v1=true"; type = types.str; }; @@ -348,25 +407,25 @@ in { }; kubeletClientCaFile = mkOption { - description = "Path to a cert file for connecting to kubelet"; + description = "Path to a cert file for connecting to kubelet."; default = null; type = types.nullOr types.path; }; kubeletClientCertFile = mkOption { - description = "Client certificate to use for connections to kubelet"; + description = "Client certificate to use for connections to kubelet."; default = null; type = types.nullOr types.path; }; kubeletClientKeyFile = mkOption { - description = "Key to use for connections to kubelet"; + description = "Key to use for connections to kubelet."; default = null; type = types.nullOr types.path; }; kubeletHttps = mkOption { - description = "Whether to use https for connections to kubelet"; + description = "Whether to use https for connections to kubelet."; default = true; type = types.bool; }; @@ -380,7 +439,7 @@ in { scheduler = { enable = mkOption { - description = "Whether to enable kubernetes scheduler."; + description = "Whether to enable Kubernetes scheduler."; default = false; type = types.bool; }; @@ -398,7 +457,7 @@ in { }; leaderElect = mkOption { - description = "Whether to start leader election before executing main loop"; + description = "Whether to start leader election before executing main loop."; type = types.bool; default = false; }; @@ -412,7 +471,7 @@ in { controllerManager = { enable = mkOption { - description = "Whether to enable kubernetes controller manager."; + description = "Whether to enable Kubernetes controller manager."; default = false; type = types.bool; }; @@ -430,7 +489,7 @@ in { }; leaderElect = mkOption { - description = "Whether to start leader election before executing main loop"; + description = "Whether to start leader election before executing main loop."; type = types.bool; default = false; }; @@ -453,12 +512,6 @@ in { type = types.nullOr types.path; }; - clusterCidr = mkOption { - description = "Kubernetes controller manager CIDR Range for Pods in cluster"; - default = "10.10.0.0/16"; - type = types.str; - }; - extraOpts = mkOption { description = "Kubernetes controller manager extra command line options."; default = ""; @@ -468,7 +521,7 @@ in { kubelet = { enable = mkOption { - description = "Whether to enable kubernetes kubelet."; + description = "Whether to enable Kubernetes kubelet."; default = false; type = types.bool; }; @@ -518,13 +571,13 @@ in { }; hostname = mkOption { - description = "Kubernetes kubelet hostname override"; + description = "Kubernetes kubelet hostname override."; default = config.networking.hostName; type = types.str; }; allowPrivileged = mkOption { - description = "Whether to allow kubernetes containers to request privileged mode."; + description = "Whether to allow Kubernetes containers to request privileged mode."; default = true; type = types.bool; }; @@ -536,7 +589,7 @@ in { }; clusterDns = mkOption { - description = "Use alternative dns."; + description = "Use alternative DNS."; default = "10.10.0.1"; type = types.str; }; @@ -548,20 +601,20 @@ in { }; networkPlugin = mkOption { - description = "Network plugin to use by kubernetes"; + description = "Network plugin to use by Kubernetes."; type = types.nullOr (types.enum ["cni" "kubenet"]); default = "kubenet"; }; cni = { packages = mkOption { - description = "List of network plugin packages to install"; + description = "List of network plugin packages to install."; type = types.listOf types.package; default = []; }; config = mkOption { - description = "Kubernetes CNI configuration"; + description = "Kubernetes CNI configuration."; type = types.listOf types.attrs; default = []; example = literalExample '' @@ -588,11 +641,29 @@ in { }; manifests = mkOption { - description = "List of manifests to bootstrap with kubelet"; + description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)"; type = types.attrsOf types.attrs; default = {}; }; + applyManifests = mkOption { + description = "Whether to apply manifests."; + default = false; + type = types.bool; + }; + + unschedulable = mkOption { + description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role."; + default = false; + type = types.bool; + }; + + taints = mkOption { + description = "."; + default = {}; + type = types.attrsOf (types.submodule [ taintOptions ]); + }; + extraOpts = mkOption { description = "Kubernetes kubelet extra command line options."; default = ""; @@ -602,7 +673,7 @@ in { proxy = { enable = mkOption { - description = "Whether to enable kubernetes proxy."; + description = "Whether to enable Kubernetes proxy."; default = false; type = types.bool; }; @@ -620,12 +691,74 @@ in { }; }; + addonManager = { + enable = mkOption { + description = "Whether to enable Kubernetes addon manager."; + default = false; + type = types.bool; + }; + + versionTag = mkOption { + description = "Version tag of Kubernetes addon manager image."; + default = "v6.4-beta.1"; + type = types.str; + }; + + addons = mkOption { + description = "Kubernetes addons (any kind of Kubernetes resource can be an addon)."; + default = { }; + type = types.attrsOf types.attrs; + example = literalExample '' + { + "my-service" = { + "apiVersion" = "v1"; + "kind" = "Service"; + "metadata" = { + "name" = "my-service"; + "namespace" = "default"; + }; + "spec" = { ... }; + }; + } + // import <nixpkgs/nixos/modules/services/cluster/kubernetes/dashboard.nix> { cfg = config.services.kubernetes; }; + ''; + }; + }; + + dns = { + enable = mkEnableOption "Kubernetes DNS service."; + + port = mkOption { + description = "Kubernetes DNS listening port."; + default = 53; + type = types.int; + }; + + domain = mkOption { + description = "Kubernetes DNS domain under which to create names."; + default = cfg.kubelet.clusterDomain; + type = types.str; + }; + + extraOpts = mkOption { + description = "Kubernetes DNS extra command line options."; + default = ""; + type = types.str; + }; + }; + path = mkOption { - description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added"; + description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added."; type = types.listOf types.package; default = []; }; + clusterCidr = mkOption { + description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster."; + default = "10.10.0.0/16"; + type = types.str; + }; + }; ###### implementation @@ -645,7 +778,10 @@ in { serviceConfig = { Slice = "kubernetes.slice"; ExecStart = ''${cfg.package}/bin/kubelet \ - --pod-manifest-path=${manifests} \ + ${optionalString cfg.kubelet.applyManifests + "--pod-manifest-path=${manifests}"} \ + ${optionalString (taints != "") + "--register-with-taints=${taints}"} \ --kubeconfig=${kubeconfig} \ --require-kubeconfig \ --address=${cfg.kubelet.address} \ @@ -677,15 +813,24 @@ in { }; }; + # Allways include cni plugins + services.kubernetes.kubelet.cni.packages = [pkgs.cni]; + }) + + (mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) { environment.etc = mapAttrs' (name: manifest: nameValuePair "kubernetes/manifests/${name}.json" { text = builtins.toJSON manifest; mode = "0755"; } ) cfg.kubelet.manifests; + }) - # Allways include cni plugins - services.kubernetes.kubelet.cni.packages = [pkgs.cni]; + (mkIf (cfg.kubelet.unschedulable && cfg.kubelet.enable) { + services.kubernetes.kubelet.taints.unschedulable = { + value = "true"; + effect = "NoSchedule"; + }; }) (mkIf cfg.apiserver.enable { @@ -728,7 +873,7 @@ in { --authorization-mode=${concatStringsSep "," cfg.apiserver.authorizationMode} \ ${optionalString (elem "ABAC" cfg.apiserver.authorizationMode) "--authorization-policy-file=${ - pkgs.writeText "kube-auth-policy" + pkgs.writeText "kube-auth-policy.jsonl" (concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy) }" } \ @@ -743,7 +888,7 @@ in { "--service-account-key-file=${cfg.apiserver.serviceAccountKeyFile}"} \ ${optionalString cfg.verbose "--v=6"} \ ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ - --storage-backend=etcd2 \ + --storage-backend=${cfg.apiserver.storageBackend} \ ${cfg.apiserver.extraOpts} ''; WorkingDirectory = cfg.dataDir; @@ -799,8 +944,8 @@ in { ${if (cfg.controllerManager.rootCaFile!=null) then "--root-ca-file=${cfg.controllerManager.rootCaFile}" else "--root-ca-file=/var/run/kubernetes/apiserver.crt"} \ - ${optionalString (cfg.controllerManager.clusterCidr!=null) - "--cluster-cidr=${cfg.controllerManager.clusterCidr}"} \ + ${optionalString (cfg.clusterCidr!=null) + "--cluster-cidr=${cfg.clusterCidr}"} \ --allocate-node-cidrs=true \ ${optionalString cfg.verbose "--v=6"} \ ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ @@ -819,7 +964,7 @@ in { description = "Kubernetes Proxy Service"; wantedBy = [ "kubernetes.target" ]; after = [ "kube-apiserver.service" ]; - path = [pkgs.iptables]; + path = [pkgs.iptables pkgs.conntrack_tools]; serviceConfig = { Slice = "kubernetes.slice"; ExecStart = ''${cfg.package}/bin/kube-proxy \ @@ -827,6 +972,8 @@ in { --bind-address=${cfg.proxy.address} \ ${optionalString cfg.verbose "--v=6"} \ ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ + ${optionalString (cfg.clusterCidr!=null) + "--cluster-cidr=${cfg.clusterCidr}"} \ ${cfg.proxy.extraOpts} ''; WorkingDirectory = cfg.dataDir; @@ -842,12 +989,18 @@ in { virtualisation.docker.enable = mkDefault true; services.kubernetes.kubelet.enable = mkDefault true; services.kubernetes.kubelet.allowPrivileged = mkDefault true; + services.kubernetes.kubelet.applyManifests = mkDefault true; services.kubernetes.apiserver.enable = mkDefault true; services.kubernetes.scheduler.enable = mkDefault true; services.kubernetes.controllerManager.enable = mkDefault true; + services.kubernetes.dns.enable = mkDefault true; services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]); }) + (mkIf (all (el: el == "master") cfg.roles) { + services.kubernetes.kubelet.unschedulable = mkDefault true; + }) + (mkIf (any (el: el == "node") cfg.roles) { virtualisation.docker.enable = mkDefault true; virtualisation.docker.logDriver = mkDefault "json-file"; @@ -855,12 +1008,42 @@ in { services.kubernetes.proxy.enable = mkDefault true; }) + (mkIf cfg.addonManager.enable { + services.kubernetes.kubelet.manifests = import ./kube-addon-manager.nix { inherit cfg addons; }; + environment.etc."kubernetes/addons".source = "${addons}/"; + }) + + (mkIf cfg.dns.enable { + systemd.services.kube-dns = { + description = "Kubernetes DNS Service"; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; + serviceConfig = { + Slice = "kubernetes.slice"; + ExecStart = ''${pkgs.kube-dns}/bin/kube-dns \ + --kubecfg-file=${kubeconfig} \ + --dns-port=${toString cfg.dns.port} \ + --domain=${cfg.dns.domain} \ + ${optionalString cfg.verbose "--v=6"} \ + ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ + ${cfg.dns.extraOpts} + ''; + WorkingDirectory = cfg.dataDir; + User = "kubernetes"; + Group = "kubernetes"; + AmbientCapabilities = "cap_net_bind_service"; + SendSIGHUP = true; + }; + }; + }) + (mkIf ( cfg.apiserver.enable || cfg.scheduler.enable || cfg.controllerManager.enable || cfg.kubelet.enable || - cfg.proxy.enable + cfg.proxy.enable || + cfg.dns.enable ) { systemd.targets.kubernetes = { description = "Kubernetes"; diff --git a/nixos/modules/services/cluster/kubernetes/dns.nix b/nixos/modules/services/cluster/kubernetes/dns.nix new file mode 100644 index 00000000000..ac59eaf8725 --- /dev/null +++ b/nixos/modules/services/cluster/kubernetes/dns.nix @@ -0,0 +1,240 @@ +{ cfg }: { + "kubedns-cm" = { + "apiVersion" = "v1"; + "kind" = "ConfigMap"; + "metadata" = { + "labels" = { + "addonmanager.kubernetes.io/mode" = "EnsureExists"; + }; + "name" = "kube-dns"; + "namespace" = "kube-system"; + }; + }; + "kubedns-controller" = { + "apiVersion" = "extensions/v1beta1"; + "kind" = "Deployment"; + "metadata" = { + "labels" = { + "addonmanager.kubernetes.io/mode" = "Reconcile"; + "k8s-app" = "kube-dns"; + "kubernetes.io/cluster-service" = "true"; + }; + "name" = "kube-dns"; + "namespace" = "kube-system"; + }; + "spec" = { + "selector" = { + "matchLabels" = { + "k8s-app" = "kube-dns"; + }; + }; + "strategy" = { + "rollingUpdate" = { + "maxSurge" = "10%"; + "maxUnavailable" = 0; + }; + }; + "template" = { + "metadata" = { + "annotations" = { + "scheduler.alpha.kubernetes.io/critical-pod" = ""; + }; + "labels" = { + "k8s-app" = "kube-dns"; + }; + }; + "spec" = { + "containers" = [{ + "args" = ["--domain=${cfg.dns.domain}." + "--dns-port=10053" + "--config-dir=/kube-dns-config" + "--kube-master-url=${cfg.kubeconfig.server}" + "--v=2" + ]; + "env" = [{ + "name" = "PROMETHEUS_PORT"; + "value" = "10055"; + }]; + "image" = "gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.1"; + "livenessProbe" = { + "failureThreshold" = 5; + "httpGet" = { + "path" = "/healthcheck/kubedns"; + "port" = 10054; + "scheme" = "HTTP"; + }; + "initialDelaySeconds" = 60; + "successThreshold" = 1; + "timeoutSeconds" = 5; + }; + "name" = "kubedns"; + "ports" = [{ + "containerPort" = 10053; + "name" = "dns-local"; + "protocol" = "UDP"; + } { + "containerPort" = 10053; + "name" = "dns-tcp-local"; + "protocol" = "TCP"; + } { + "containerPort" = 10055; + "name" = "metrics"; + "protocol" = "TCP"; + }]; + "readinessProbe" = { + "httpGet" = { + "path" = "/readiness"; + "port" = 8081; + "scheme" = "HTTP"; + }; + "initialDelaySeconds" = 3; + "timeoutSeconds" = 5; + }; + "resources" = { + "limits" = { + "memory" = "170Mi"; + }; + "requests" = { + "cpu" = "100m"; + "memory" = "70Mi"; + }; + }; + "volumeMounts" = [{ + "mountPath" = "/kube-dns-config"; + "name" = "kube-dns-config"; + }]; + } { + "args" = ["-v=2" + "-logtostderr" + "-configDir=/etc/k8s/dns/dnsmasq-nanny" + "-restartDnsmasq=true" + "--" + "-k" + "--cache-size=1000" + "--log-facility=-" + "--server=/${cfg.dns.domain}/127.0.0.1#10053" + "--server=/in-addr.arpa/127.0.0.1#10053" + "--server=/ip6.arpa/127.0.0.1#10053" + ]; + "image" = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.1"; + "livenessProbe" = { + "failureThreshold" = 5; + "httpGet" = { + "path" = "/healthcheck/dnsmasq"; + "port" = 10054; + "scheme" = "HTTP"; + }; + "initialDelaySeconds" = 60; + "successThreshold" = 1; + "timeoutSeconds" = 5; + }; + "name" = "dnsmasq"; + "ports" = [{ + "containerPort" = 53; + "name" = "dns"; + "protocol" = "UDP"; + } { + "containerPort" = 53; + "name" = "dns-tcp"; + "protocol" = "TCP"; + }]; + "resources" = { + "requests" = { + "cpu" = "150m"; + "memory" = "20Mi"; + }; + }; + "volumeMounts" = [{ + "mountPath" = "/etc/k8s/dns/dnsmasq-nanny"; + "name" = "kube-dns-config"; + }]; + } { + "args" = ["--v=2" + "--logtostderr" + "--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.${cfg.dns.domain},5,A" + "--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.${cfg.dns.domain},5,A" + ]; + "image" = "gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.1"; + "livenessProbe" = { + "failureThreshold" = 5; + "httpGet" = { + "path" = "/metrics"; + "port" = 10054; + "scheme" = "HTTP"; + }; + "initialDelaySeconds" = 60; + "successThreshold" = 1; + "timeoutSeconds" = 5; + }; + "name" = "sidecar"; + "ports" = [{ + "containerPort" = 10054; + "name" = "metrics"; + "protocol" = "TCP"; + }]; + "resources" = { + "requests" = { + "cpu" = "10m"; + "memory" = "20Mi"; + }; + }; + }]; + "dnsPolicy" = "Default"; + "serviceAccountName" = "kube-dns"; + "tolerations" = [{ + "key" = "CriticalAddonsOnly"; + "operator" = "Exists"; + }]; + "volumes" = [{ + "configMap" = { + "name" = "kube-dns"; + "optional" = true; + }; + "name" = "kube-dns-config"; + }]; + }; + }; + }; + }; + "kubedns-sa" = { + "apiVersion" = "v1"; + "kind" = "ServiceAccount"; + "metadata" = { + "labels" = { + "addonmanager.kubernetes.io/mode" = "Reconcile"; + "kubernetes.io/cluster-service" = "true"; + }; + "name" = "kube-dns"; + "namespace" = "kube-system"; + }; + }; + "kubedns-svc" = { + "apiVersion" = "v1"; + "kind" = "Service"; + "metadata" = { + "labels" = { + "addonmanager.kubernetes.io/mode" = "Reconcile"; + "k8s-app" = "kube-dns"; + "kubernetes.io/cluster-service" = "true"; + "kubernetes.io/name" = "KubeDNS"; + }; + "name" = "kube-dns"; + "namespace" = "kube-system"; + }; + "spec" = { + "clusterIP" = "${cfg.dns.serverIp}"; + "ports" = [{ + "name" = "dns"; + "port" = 53; + "protocol" = "UDP"; + } { + "name" = "dns-tcp"; + "port" = 53; + "protocol" = "TCP"; + }]; + "selector" = { + "k8s-app" = "kube-dns"; + }; + }; + }; +} diff --git a/nixos/modules/services/cluster/kubernetes/kube-addon-manager.nix b/nixos/modules/services/cluster/kubernetes/kube-addon-manager.nix new file mode 100644 index 00000000000..f1a367dfff1 --- /dev/null +++ b/nixos/modules/services/cluster/kubernetes/kube-addon-manager.nix @@ -0,0 +1,54 @@ +{ cfg, addons }: { + "kube-addon-manager" = { + "apiVersion" = "v1"; + "kind" = "Pod"; + "metadata" = { + "labels" = { + "component" = "kube-addon-manager"; + }; + "name" = "kube-addon-manager"; + "namespace" = "kube-system"; + }; + "spec" = { + "containers" = [{ + "command" = ["/bin/bash" + "-c" + "/opt/kube-addons.sh | tee /var/log/kube-addon-manager.log" + ]; + "env" = [{ + "name" = "KUBECTL_OPTS"; + "value" = "--server=${cfg.kubeconfig.server}"; + }]; + "image" = "gcr.io/google-containers/kube-addon-manager:${cfg.addonManager.versionTag}"; + "name" = "kube-addon-manager"; + "resources" = { + "requests" = { + "cpu" = "5m"; + "memory" = "50Mi"; + }; + }; + "volumeMounts" = [{ + "mountPath" = "/etc/kubernetes/addons/"; + "name" = "addons"; + "readOnly" = true; + } { + "mountPath" = "/var/log"; + "name" = "varlog"; + "readOnly" = false; + }]; + }]; + "hostNetwork" = true; + "volumes" = [{ + "hostPath" = { + "path" = "${addons}/"; + }; + "name" = "addons"; + } { + "hostPath" = { + "path" = "/var/log"; + }; + "name" = "varlog"; + }]; + }; + }; +} |