fluentd: 0.14.0 -> 0.14.11, with more plugins

I added the most popular plugins (S3, kafka, kinesis, mongo, etc.)
and tried to bring it closer to the set of plugins they bundle with
td-agent, without getting rid of the ones we already have today. We
should probably make this overridable more generically going forward
but this should scratch most itches today.

3 files changed, 401 insertions, 57 deletions

diff --git a/pkgs/tools/misc/fluentd/Gemfile b/pkgs/tools/misc/fluentd/Gemfile
index 8c9dd3aa0a0..2c4fbc84963 100644
--- a/pkgs/tools/misc/fluentd/Gemfile
+++ b/pkgs/tools/misc/fluentd/Gemfile
@@ -3,3 +3,12 @@ source "https://rubygems.org"
 gem 'fluentd'
 gem 'fluent-plugin-elasticsearch'
 gem 'fluent-plugin-record-reformer'
+gem 'fluent-plugin-s3'
+gem 'fluent-plugin-kinesis'
+gem 'fluent-plugin-kafka'
+gem 'fluent-plugin-elasticsearch'
+gem 'fluent-plugin-scribe'
+gem 'fluent-plugin-mongo'
+gem 'fluent-plugin-webhdfs'
+gem 'fluent-plugin-rewrite-tag-filter'
+
diff --git a/pkgs/tools/misc/fluentd/Gemfile.lock b/pkgs/tools/misc/fluentd/Gemfile.lock
index 581fa6e169a..2f9485d9577 100644
--- a/pkgs/tools/misc/fluentd/Gemfile.lock
+++ b/pkgs/tools/misc/fluentd/Gemfile.lock
@@ -1,58 +1,136 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    cool.io (1.4.4)
-    elasticsearch (1.0.17)
-      elasticsearch-api (= 1.0.17)
-      elasticsearch-transport (= 1.0.17)
-    elasticsearch-api (1.0.17)
+    activesupport (5.0.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    addressable (2.5.0)
+      public_suffix (~> 2.0, >= 2.0.2)
+    aws-sdk (2.7.0)
+      aws-sdk-resources (= 2.7.0)
+    aws-sdk-core (2.7.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.7.0)
+      aws-sdk-core (= 2.7.0)
+    aws-sigv4 (1.0.0)
+    bson (1.12.5)
+    bzip2-ffi (1.0.0)
+      ffi (~> 1.0)
+    concurrent-ruby (1.0.4)
+    cool.io (1.4.5)
+    elasticsearch (1.0.18)
+      elasticsearch-api (= 1.0.18)
+      elasticsearch-transport (= 1.0.18)
+    elasticsearch-api (1.0.18)
       multi_json
-    elasticsearch-transport (1.0.17)
+    elasticsearch-transport (1.0.18)
       faraday
       multi_json
-    excon (0.49.0)
-    faraday (0.9.2)
+    excon (0.54.0)
+    faraday (0.11.0)
       multipart-post (>= 1.2, < 3)
-    fluent-plugin-elasticsearch (1.5.0)
-      elasticsearch
+    ffi (1.9.17)
+    fluent-mixin-config-placeholders (0.4.0)
+      fluentd
+      uuidtools (>= 2.1.5)
+    fluent-mixin-plaintextformatter (0.2.6)
+      fluentd
+      ltsv
+    fluent-plugin-elasticsearch (1.9.2)
+      elasticsearch (< 1.1)
       excon
       fluentd (>= 0.10.43)
-    fluent-plugin-record-reformer (0.8.1)
+    fluent-plugin-kafka (0.5.0)
+      fluentd (>= 0.10.58, < 2)
+      ltsv
+      ruby-kafka (= 0.3.16.beta2)
+    fluent-plugin-kinesis (1.1.2)
+      aws-sdk (~> 2)
+      concurrent-ruby (~> 1)
+      fluentd (>= 0.10.58, < 2)
+      os (>= 0.9.6)
+      protobuf (>= 3.5.5)
+    fluent-plugin-mongo (0.7.16)
+      fluentd (>= 0.10.58, < 2)
+      mongo (~> 1.9)
+    fluent-plugin-record-reformer (0.8.2)
+      fluentd
+    fluent-plugin-rewrite-tag-filter (1.5.5)
       fluentd
-    fluentd (0.14.0)
-      cool.io (>= 1.4.3, < 2.0.0)
+    fluent-plugin-s3 (0.8.0)
+      aws-sdk (>= 2.3.22, < 3)
+      fluentd (>= 0.10.58, < 2)
+    fluent-plugin-scribe (0.10.14)
+      fluentd
+      thrift (~> 0.8.0)
+    fluent-plugin-webhdfs (0.5.2)
+      bzip2-ffi
+      fluent-mixin-config-placeholders (>= 0.3.0)
+      fluent-mixin-plaintextformatter (>= 0.2.1)
+      fluentd (>= 0.10.59)
+      webhdfs (>= 0.6.0)
+    fluentd (0.14.11)
+      cool.io (~> 1.4.5)
       http_parser.rb (>= 0.5.1, < 0.7.0)
-      json (>= 1.4.3)
-      msgpack (>= 0.7.0)
-      serverengine (>= 1.6.4)
+      msgpack (>= 0.7.0, < 2.0.0)
+      serverengine (>= 2.0.4, < 3.0.0)
       sigdump (~> 0.2.2)
-      strptime (>= 0.1.7)
-      tzinfo (>= 1.0.0)
-      tzinfo-data (>= 1.0.0)
+      strptime (~> 0.1.7)
+      tzinfo (~> 1.0)
+      tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
     http_parser.rb (0.6.0)
-    json (1.8.3)
-    msgpack (0.7.6)
+    i18n (0.7.0)
+    jmespath (1.3.1)
+    ltsv (0.1.0)
+    middleware (0.1.0)
+    minitest (5.10.1)
+    mongo (1.12.5)
+      bson (= 1.12.5)
+    msgpack (1.0.2)
     multi_json (1.12.1)
     multipart-post (2.0.0)
-    serverengine (1.6.4)
+    os (0.9.6)
+    protobuf (3.6.12)
+      activesupport (>= 3.2)
+      middleware
+      thor
+      thread_safe
+    public_suffix (2.0.5)
+    ruby-kafka (0.3.16.beta2)
+    serverengine (2.0.4)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    strptime (0.1.8)
+    strptime (0.1.9)
+    thor (0.19.4)
     thread_safe (0.3.5)
+    thrift (0.8.0)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2016.4)
+    tzinfo-data (1.2016.10)
       tzinfo (>= 1.0.0)
-    yajl-ruby (1.2.1)
+    uuidtools (2.1.5)
+    webhdfs (0.8.0)
+      addressable
+    yajl-ruby (1.3.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   fluent-plugin-elasticsearch
+  fluent-plugin-kafka
+  fluent-plugin-kinesis
+  fluent-plugin-mongo
   fluent-plugin-record-reformer
+  fluent-plugin-rewrite-tag-filter
+  fluent-plugin-s3
+  fluent-plugin-scribe
+  fluent-plugin-webhdfs
   fluentd
 
 BUNDLED WITH
-   1.11.2
+   1.12.5
diff --git a/pkgs/tools/misc/fluentd/gemset.nix b/pkgs/tools/misc/fluentd/gemset.nix
index e6b03fadfd3..1c508e7b58e 100644
--- a/pkgs/tools/misc/fluentd/gemset.nix
+++ b/pkgs/tools/misc/fluentd/gemset.nix
@@ -1,75 +1,227 @@
 {
+  activesupport = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "08bnl0nr9csjgkgz6xf8dyg7rccinmfrmn235z3bfaz8ihz15d1d";
+      type = "gem";
+    };
+    version = "5.0.1";
+  };
+  addressable = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "1j5r0anj8m4qlf2psnldip4b8ha2bsscv11lpdgnfh4nnchzjnxw";
+      type = "gem";
+    };
+    version = "2.5.0";
+  };
+  aws-sdk = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "19s7ialas1yrc54g50yfa37z7m8dq4gqbf8dvlfg8qmpdijjxy3l";
+      type = "gem";
+    };
+    version = "2.7.0";
+  };
+  aws-sdk-core = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0a9sgff43s3zhpcmisk1bp6vvlpawa617svfhz84xwa6lmik9sp4";
+      type = "gem";
+    };
+    version = "2.7.0";
+  };
+  aws-sdk-resources = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "1b5z25n4bgzwkzmzx2q6ik2y74jinyphmrh38lnrn9im6pmmvy3w";
+      type = "gem";
+    };
+    version = "2.7.0";
+  };
+  aws-sigv4 = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0cnrfxbaxn86qrxhfidg10f89ka1hddihakdhcvnri0dljaw7dsz";
+      type = "gem";
+    };
+    version = "1.0.0";
+  };
+  bson = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "12zcsfr72hr0w1qyxv1iz587nzganpclvimyx5y02gg1hij8hz6b";
+      type = "gem";
+    };
+    version = "1.12.5";
+  };
+  bzip2-ffi = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "1y5jlcz1vb0v3rbmsbbrarfglcmzdhr5jhlfc5wjnhz2zpybsz3y";
+      type = "gem";
+    };
+    version = "1.0.0";
+  };
+  concurrent-ruby = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0p7ji1h1l407kci9w4b4yspzd58ssmlx7p91npx55kw08836dlpb";
+      type = "gem";
+    };
+    version = "1.0.4";
+  };
   "cool.io" = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "0ycc8qdvpba8bf6da8nsna34md86mk527j4qizxh059vqm3521sb";
+      sha256 = "1x5fkyjdjwk68sg7fwxhx2k3hzxkkm6frnd2yix7brxdh06fp0k1";
       type = "gem";
     };
-    version = "1.4.4";
+    version = "1.4.5";
   };
   elasticsearch = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "1g7vax396l68w5mrrfbsaly39zkc4rrvljz9717mxyn82m5f66w5";
+      sha256 = "1wdy17i56b4m7akp7yavnr8vhfhyz720waphmixq05dj21b11hl0";
       type = "gem";
     };
-    version = "1.0.17";
+    version = "1.0.18";
   };
   elasticsearch-api = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "08bb63raz381fmspijwjc4ksvrrgavmwrymjms1b9mg4qkic87jx";
+      sha256 = "1v6nb3ajz5rack3p4b4nz37hs0zb9x738h2ms8cc4plp6wqh1w5s";
       type = "gem";
     };
-    version = "1.0.17";
+    version = "1.0.18";
   };
   elasticsearch-transport = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "07r798g3lnzr3zabk2ks2j5jnxdga23bc8wrr7mcqzn8q0yv82bz";
+      sha256 = "0smfrz8nq49hgf67y5ayxa9i4rmmi0q4m51l0h499ykq4cvcwv6i";
       type = "gem";
     };
-    version = "1.0.17";
+    version = "1.0.18";
   };
   excon = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "0jmdgc4lhlbxccpg79a32vn3qngqipcaaq8bxa0ivfw5mvz0zc0z";
+      sha256 = "0j4b6s90v84r4wrhbg4rzjfjg9sfisq50fjd3hh9p6yrkm86wbd3";
       type = "gem";
     };
-    version = "0.49.0";
+    version = "0.54.0";
   };
   faraday = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "1kplqkpn2s2yl3lxdf6h7sfldqvkbkpxwwxhyk7mdhjplb5faqh6";
+      sha256 = "18p1csdivgwmshfw3mb698a3bn0yrykg30khk5qxjf6n168g91jr";
+      type = "gem";
+    };
+    version = "0.11.0";
+  };
+  ffi = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "07hnyr47mndsjfanzh348wm3fxjx9nx68mdb3cpsdvfqrxnz97s7";
+      type = "gem";
+    };
+    version = "1.9.17";
+  };
+  fluent-mixin-config-placeholders = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "14b4lqy91jgpky6g7h0vyfy2rr1qavmjzzgnmhwajfzxgw9y2jvi";
+      type = "gem";
+    };
+    version = "0.4.0";
+  };
+  fluent-mixin-plaintextformatter = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0gliangfr07060ya9sawkyfx2vz7vdygys65f83czawhckvvm75n";
       type = "gem";
     };
-    version = "0.9.2";
+    version = "0.2.6";
   };
   fluent-plugin-elasticsearch = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "1kgv62s51y9x98qk0b6wrg4a73jfbhw50vg5z36hr0bh9rh2rq4y";
+      sha256 = "0q0v8jxpwrkh1z5qh0chwrssz93nldka4jwfn32hlqhnmb99q8i1";
+      type = "gem";
+    };
+    version = "1.9.2";
+  };
+  fluent-plugin-kafka = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0sd025xsl1cnjs11wasg0di2k02rx9ifaj49n28ak363df6vsqgf";
+      type = "gem";
+    };
+    version = "0.5.0";
+  };
+  fluent-plugin-kinesis = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "119ngswi9q0p5hh5ldan9pzrgd1lfsbkr5f56hy1k4gfss4kmq27";
+      type = "gem";
+    };
+    version = "1.1.2";
+  };
+  fluent-plugin-mongo = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "1x7n8cknqh956yx3c9hv2g535x4kcixmnxw3fvcspjbqprrd1s91";
       type = "gem";
     };
-    version = "1.5.0";
+    version = "0.7.16";
   };
   fluent-plugin-record-reformer = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "1ca09msvcdgrjv0xdjxh0nhxx8crp3h9nz5qw90c75s5hss2ws9b";
+      sha256 = "1q2pws1mqp6pkb00ix6wjkxklckqb4wcbp79lpyk0b644bk9hqzb";
+      type = "gem";
+    };
+    version = "0.8.2";
+  };
+  fluent-plugin-rewrite-tag-filter = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "1avxvvmfm7bl7fpa2p73295kydh1nbsgdvsr7bsyrb77z1s1m86z";
       type = "gem";
     };
-    version = "0.8.1";
+    version = "1.5.5";
+  };
+  fluent-plugin-s3 = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0nxvk5n76pw4r37lv8vfl1cd0yjxnlj5wlwyk8f1lvp9ma5zlzmg";
+      type = "gem";
+    };
+    version = "0.8.0";
+  };
+  fluent-plugin-scribe = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "00m19w7p22adq0yx1h7h2h4ckw9kh5j458a8lawgmbazw2dz0zxi";
+      type = "gem";
+    };
+    version = "0.10.14";
+  };
+  fluent-plugin-webhdfs = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0kb9cgrgvh61pqqzv2csnibmp2jwh4hyjyvrh2npkk59k3jp54ad";
+      type = "gem";
+    };
+    version = "0.5.2";
   };
   fluentd = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "1v6c8g6fv9s710lrl0jy9ihbb8af37gvw3klk7csr5whp1mhwb8f";
+      sha256 = "0w1bg3nrn6gwhyp8xlpbs9rcajkddnvw6jhn7kvzydp70g2aydhz";
       type = "gem";
     };
-    version = "0.14.0";
+    version = "0.14.11";
   };
   "http_parser.rb" = {
     source = {
@@ -78,21 +230,61 @@
     };
     version = "0.6.0";
   };
-  json = {
+  i18n = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "1i5z1ykl8zhszsxcs8mzl8d0dxgs3ylz8qlzrw74jb0gplkx6758";
+      type = "gem";
+    };
+    version = "0.7.0";
+  };
+  jmespath = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "07w8ipjg59qavijq59hl82zs74jf3jsp7vxl9q3a2d0wpv5akz3y";
+      type = "gem";
+    };
+    version = "1.3.1";
+  };
+  ltsv = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "1alfq3g0mih4w86736ybnzqmknphm2z95c9q0wl765i4lrmxng11";
+      type = "gem";
+    };
+    version = "0.1.0";
+  };
+  middleware = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0703nkf2v371wqr41c04x5qid7ww45cxqv3hnlg07if3b3xrm9xl";
+      type = "gem";
+    };
+    version = "0.1.0";
+  };
+  minitest = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "1yk2m8sp0p5m1niawa3ncg157a4i0594cg7z91rzjxv963rzrwab";
+      type = "gem";
+    };
+    version = "5.10.1";
+  };
+  mongo = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "1nsby6ry8l9xg3yw4adlhk2pnc7i0h0rznvcss4vk3v74qg0k8lc";
+      sha256 = "0658pn2hbyfvbnpp3wdh3irin0wpikm6y2qbhnx07w54jbkmgh5p";
       type = "gem";
     };
-    version = "1.8.3";
+    version = "1.12.5";
   };
   msgpack = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "1fn2riiaygiyvmr0glgm1vx995np3jb2hjf5i0j78vncd2wbwdw5";
+      sha256 = "1fb2my91j08plsbbry5kilsrh7slmzgbbf6f55zy6xk28p9036lg";
       type = "gem";
     };
-    version = "0.7.6";
+    version = "1.0.2";
   };
   multi_json = {
     source = {
@@ -109,13 +301,45 @@
     };
     version = "2.0.0";
   };
+  os = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "1llv8w3g2jwggdxr5a5cjkrnbbfnvai3vxacxxc0fy84xmz3hymz";
+      type = "gem";
+    };
+    version = "0.9.6";
+  };
+  protobuf = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0cvkfp7574dr6wqpgafl3pg9niqfri3dh2fxb2f8qaapcgfgcaq6";
+      type = "gem";
+    };
+    version = "3.6.12";
+  };
+  public_suffix = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "040jf98jpp6w140ghkhw2hvc1qx41zvywx5gj7r2ylr1148qnj7q";
+      type = "gem";
+    };
+    version = "2.0.5";
+  };
+  ruby-kafka = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "112avql9nf0hq07zvh47fyx7na721bj4zfpf43ip471l6k2ksrf5";
+      type = "gem";
+    };
+    version = "0.3.16.beta2";
+  };
   serverengine = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "16sy6yissv8h2vla5ba4msqzsjy0cm0x8q2llssx3kl3bwysrbrp";
+      sha256 = "0f08kbiqg9yp5fxdw5blsrnq383a9g4n830g1ypppb7ddv61sbmi";
       type = "gem";
     };
-    version = "1.6.4";
+    version = "2.0.4";
   };
   sigdump = {
     source = {
@@ -128,10 +352,18 @@
   strptime = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "0lkadizgdls9ya4sbf3bg5i1z6g2kxfw1r5ja0wkc9711zxjilx2";
+      sha256 = "1avbl1fj4y5qx9ywkxpcjjxxpjj6h7r1dqlnddhk5wqg6ypq8lsb";
+      type = "gem";
+    };
+    version = "0.1.9";
+  };
+  thor = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "01n5dv9kql60m6a00zc0r66jvaxx98qhdny3klyj0p3w34pad2ns";
       type = "gem";
     };
-    version = "0.1.8";
+    version = "0.19.4";
   };
   thread_safe = {
     source = {
@@ -140,6 +372,14 @@
     };
     version = "0.3.5";
   };
+  thrift = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0aj23ndh5n5yqcvp4c12y7vl5wvxpl66zncf6n6ax2zvb6ig44cv";
+      type = "gem";
+    };
+    version = "0.8.0";
+  };
   tzinfo = {
     dependencies = ["thread_safe"];
     source = {
@@ -151,16 +391,33 @@
   tzinfo-data = {
     source = {
       remotes = ["https://rubygems.org"];
-      sha256 = "1bxfljd5i7g89s7jc5l4a3ddykfsvvp0gm02805r1q77ahn1gp33";
+      sha256 = "01nr50alfm1fyzlcbzvfbpnsq37yb3h676f9n3z0iyp4s4766psf";
+      type = "gem";
+    };
+    version = "1.2016.10";
+  };
+  uuidtools = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0zjvq1jrrnzj69ylmz1xcr30skf9ymmvjmdwbvscncd7zkr8av5g";
+      type = "gem";
+    };
+    version = "2.1.5";
+  };
+  webhdfs = {
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0gs6xb9dz9bp5xc38yplfy48jcgj7jrj0zg0vgi7ydkxnkzkhbf2";
       type = "gem";
     };
-    version = "1.2016.4";
+    version = "0.8.0";
   };
   yajl-ruby = {
     source = {
-      sha256 = "0zvvb7i1bl98k3zkdrnx9vasq0rp2cyy5n7p9804dqs4fz9xh9vf";
+      remotes = ["https://rubygems.org"];
+      sha256 = "0sah2lpvpsh555dcnhgcqylinjj5544md9dh1a0a13da0qv1p57i";
       type = "gem";
     };
-    version = "1.2.1";
+    version = "1.3.0";
   };
 }
\ No newline at end of file