osquery: 3.2.9 -> 3.3.2

Although there's already a windows-only 3.4[1], the latest release for
Unix systems is 3.3.2[2] with the following changes:

* SELinux event recording[3]
* SMART drive information[4]
* ELF parsing to monitor compiler-level protections for binaries[5]

The diff became rather big in the end due to the following changes:

* The SMART information feature uses a custom fork of smartmontools to
  retrieve SMART data programatically[6].

* Replaced the old, bloated `misc.patch` by a simpler patch that only
  fixes two (actual) issues in the CMake build, unnecessary link targets
  and an invalid compiler flag.

* I tried to clean the derivation's code up a little bit by
  simplifying the `preConfigure` hook and adding only one build-input
  per line (that makes merging of several changes to the derivation
  easier as well).

[1] https://github.com/facebook/osquery/releases/tag/3.4.0
[2] https://github.com/facebook/osquery/releases/tag/3.3.2
[3] https://github.com/facebook/osquery/pull/4224
[4] https://github.com/facebook/osquery/pull/4133
[5] https://github.com/facebook/osquery/pull/4708
[6] https://github.com/allanliu/smartmontools

3 files changed, 140 insertions, 238 deletions

diff --git a/pkgs/tools/system/osquery/0001-Fix-CMake-configuration-for-Nix.patch b/pkgs/tools/system/osquery/0001-Fix-CMake-configuration-for-Nix.patch
new file mode 100644
index 00000000000..b90752f8dc5
--- /dev/null
+++ b/pkgs/tools/system/osquery/0001-Fix-CMake-configuration-for-Nix.patch
@@ -0,0 +1,41 @@
+From 511400b8ec0794588df88e2bb1c45cc6a4a51b1e Mon Sep 17 00:00:00 2001
+From: Maximilian Bosch <maximilian@mbosch.me>
+Date: Sun, 26 May 2019 23:10:02 +0200
+Subject: [PATCH] Fix CMake configuration for Nix
+
+* -fno-limit-debug-info: CLang specific, patch away to support both
+  CLang and GCC.
+* Dropped unnecessary `lvm-internal` and `daemonclient` references.
+---
+ CMakeLists.txt                | 1 -
+ osquery/tables/CMakeLists.txt | 2 +-
+ 2 files changed, 1 insertion(+), 2 deletions(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index f545d6c9..cfe68ddb 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -328,7 +328,6 @@ if (POSIX)
+   # To be safe, only include them when building 'release' outputs.
+   add_compile_options(
+     "$<$<CONFIG:RELEASE>:-g>"
+-    "$<$<CONFIG:RELEASE>:-fno-limit-debug-info>"
+     "$<$<CONFIG:RELEASE>:-fPIE>"
+     "$<$<CONFIG:RELEASE>:-fpie>"
+     "$<$<CONFIG:RELEASE>:-fPIC>"
+diff --git a/osquery/tables/CMakeLists.txt b/osquery/tables/CMakeLists.txt
+index 470bdf1d..caa6ab6f 100644
+--- a/osquery/tables/CMakeLists.txt
++++ b/osquery/tables/CMakeLists.txt
+@@ -68,7 +68,7 @@ if(LINUX)
+   set(TABLE_PLATFORM "linux")
+ 
+   ADD_OSQUERY_LINK_ADDITIONAL("libresolv.so")
+-  ADD_OSQUERY_LINK_ADDITIONAL("cryptsetup devmapper lvm2app lvm-internal daemonclient")
++  ADD_OSQUERY_LINK_ADDITIONAL("cryptsetup devmapper lvm2app")
+   ADD_OSQUERY_LINK_ADDITIONAL("gcrypt gpg-error")
+   ADD_OSQUERY_LINK_ADDITIONAL("blkid")
+   ADD_OSQUERY_LINK_ADDITIONAL("ip4tc")
+-- 
+2.19.2
+
diff --git a/pkgs/tools/system/osquery/default.nix b/pkgs/tools/system/osquery/default.nix
index 1c1da724676..84860df979c 100644
--- a/pkgs/tools/system/osquery/default.nix
+++ b/pkgs/tools/system/osquery/default.nix
@@ -1,58 +1,20 @@
-{ stdenv, lib, fetchFromGitHub, pkgconfig, cmake, pythonPackages
+{ stdenv, lib, fetchFromGitHub, pkgconfig, cmake, python
 , udev, audit, aws-sdk-cpp, cryptsetup, lvm2, libgcrypt, libarchive
 , libgpgerror, libuuid, iptables, dpkg, lzma, bzip2, rpm
 , beecrypt, augeas, libxml2, sleuthkit, yara, lldpd, google-gflags
 , thrift, boost, rocksdb_lite, glog, gbenchmark, snappy
 , openssl, file, doxygen
-, gtest, sqlite, fpm, zstd, rdkafka, rapidjson, fetchgit, fetchurl
+, gtest, sqlite, fpm, zstd, rdkafka, rapidjson, fetchgit, fetchurl, libelfin
+, smartmontools, which, git, cscope, ctags, ssdeep
 }:
 
 let
-
-  thirdparty = fetchFromGitHub {
-    owner = "osquery";
-    repo = "third-party";
-    rev = "32e01462fbea75d3b1904693f937dfd62eaced15";
-    sha256 = "0va24gmgk43a1lyjs63q9qrhvpv8gmqjzpjr5595vhr16idv8wyf";
-  };
-
-in
-
-stdenv.mkDerivation rec {
-  name = "osquery-${version}";
-  version = "3.2.9";
-
-  # this is what `osquery --help` will show as the version.
-  OSQUERY_BUILD_VERSION = version;
-  OSQUERY_PLATFORM = "NixOS;";
-
-  src = fetchFromGitHub {
-    owner = "facebook";
-    repo = "osquery";
-    rev = version;
-    sha256 = "1fac0yj1701469qhbsp38ab2fmavm3jw6x278bf78yvxdi99ivai";
-  };
-
-  patches = [ ./misc.patch ];
-
-  nativeBuildInputs = [
-    pkgconfig cmake pythonPackages.python pythonPackages.jinja2 doxygen fpm
-  ];
-
-  NIX_LDFLAGS = [
-    "-lcrypto"
-  ];
-
-  buildInputs = let
-    gflags' = google-gflags.overrideAttrs (old: {
-      cmakeFlags = stdenv.lib.filter (f: (builtins.match ".*STATIC.*" f) == null) old.cmakeFlags;
-    });
-
+  overrides = {
     # use older `lvm2` source for osquery, the 2.03 sourcetree
     # will break osquery due to the lacking header `lvm2app.h`.
     #
     # https://github.com/NixOS/nixpkgs/pull/51756#issuecomment-446035295
-    lvm2' = lvm2.overrideAttrs (old: rec {
+    lvm2 = lvm2.overrideAttrs (old: rec {
       name = "lvm2-${version}";
       version = "2.02.183";
       src = fetchgit {
@@ -62,8 +24,25 @@ stdenv.mkDerivation rec {
       };
     });
 
+    # use smartmontools fork to programatically retrieve SMART information.
+    # https://github.com/facebook/osquery/pull/4133
+    smartmontools = smartmontools.overrideAttrs (old: rec {
+      name = "smartmontools-${version}";
+      version = "0.3.1";
+      src = fetchFromGitHub {
+        owner = "allanliu";
+        repo = "smartmontools";
+        rev = "v${version}";
+        sha256 = "1i72fk2ranrky02h7nh9l3va4kjzj0lx1gr477zkxd44wf3w0pjf";
+      };
+
+      # Apple build fix doesn't apply here and isn't needed as we
+      # only support `osquery` on Linux.
+      patches = [];
+    });
+
     # dpkg 1.19.2 dropped api in `<dpkg/dpkg-db.h>` which breaks compilation.
-    dpkg' = dpkg.overrideAttrs (old: rec {
+    dpkg = dpkg.overrideAttrs (old: rec {
       name = "dpkg-${version}";
       version = "1.19.0.5";
       src = fetchurl {
@@ -71,33 +50,92 @@ stdenv.mkDerivation rec {
         sha256 = "1dc5kp3fqy1k66fly6jfxkkg7w6d0jy8szddpfyc2xvzga94d041";
       };
     });
-  in [
-    udev audit
 
+    # filter out static linking configuration to avoid that the library will
+    # be linked both statically and dynamically.
+    gflags = google-gflags.overrideAttrs (old: {
+      cmakeFlags = stdenv.lib.filter (f: (builtins.match ".*STATIC.*" f) == null) old.cmakeFlags;
+    });
+  };
+in
+
+stdenv.mkDerivation rec {
+  pname = "osquery";
+  version = "3.3.2";
+
+  # this is what `osquery --help` will show as the version.
+  OSQUERY_BUILD_VERSION = version;
+  OSQUERY_PLATFORM = "NixOS;";
+
+  src = fetchFromGitHub {
+    owner = "facebook";
+    repo = pname;
+    rev = version;
+    sha256 = "0nrwmzmbziacs3y0nljyc73bibr3w68myjpfwkicg9zgkq4qihij";
+  };
+
+  patches = [ ./0001-Fix-CMake-configuration-for-Nix.patch ];
+
+  NIX_CFLAGS_COMPILE = [
+    "-I${libxml2.dev}/include/libxml2"
+  ];
+
+  nativeBuildInputs = [ python which git cscope ctags cmake pkgconfig doxygen fpm ]
+    ++ (with python.pkgs; [ jinja2 ]);
+
+  buildInputs = [
+    udev
+    audit
     (aws-sdk-cpp.override {
       apis = [ "firehose" "kinesis" "sts" "ec2" ];
       customMemoryManagement = false;
     })
-
-    lvm2' libgcrypt libarchive libgpgerror libuuid iptables dpkg'
-    lzma bzip2 rpm beecrypt augeas libxml2 sleuthkit
-    yara lldpd gflags' thrift boost
-    glog gbenchmark snappy openssl
-    file cryptsetup
-    gtest sqlite zstd rdkafka rapidjson rocksdb_lite
+    overrides.lvm2
+    libgcrypt
+    libarchive
+    libgpgerror
+    libuuid
+    iptables
+    overrides.dpkg
+    lzma
+    bzip2
+    rpm
+    beecrypt
+    augeas
+    libxml2
+    sleuthkit
+    yara
+    lldpd
+    overrides.gflags
+    thrift
+    boost
+    glog
+    gbenchmark
+    snappy
+    openssl
+    file
+    cryptsetup
+    gtest
+    zstd
+    rdkafka
+    rapidjson
+    rocksdb_lite
+    libelfin
+    ssdeep
+    overrides.smartmontools
   ];
 
-  preConfigure = ''
-    export NIX_CFLAGS_COMPILE="-I${libxml2.dev}/include/libxml2 $NIX_CFLAGS_COMPILE"
+  cmakeFlags = [ "-DSKIP_TESTS=1" ];
 
-    cmakeFlagsArray+=(
-      -DCMAKE_LIBRARY_PATH=${cryptsetup}/lib
-      -DCMAKE_VERBOSE_MAKEFILE=OFF
-    )
+  preConfigure = ''
+    cp -r ${fetchFromGitHub {
+      owner = "osquery";
+      repo = "third-party";
+      rev = "32e01462fbea75d3b1904693f937dfd62eaced15";
+      sha256 = "0va24gmgk43a1lyjs63q9qrhvpv8gmqjzpjr5595vhr16idv8wyf";
+    }}/* third-party
 
-    cp -r ${thirdparty}/* third-party
     chmod +w -R third-party
-    rm -r third-party/{googletest,sqlite3}
   '';
 
   meta = with lib; {
diff --git a/pkgs/tools/system/osquery/misc.patch b/pkgs/tools/system/osquery/misc.patch
deleted file mode 100644
index 129c524521d..00000000000
--- a/pkgs/tools/system/osquery/misc.patch
+++ /dev/null
@@ -1,177 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 0242fb71..5007aace 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -304,7 +304,6 @@ else()
-     # To be safe, only include them when building 'release' outputs.
-     add_compile_options(
-       -g
--      -fno-limit-debug-info
-       -fPIE
-       -fpie
-       -fPIC
-@@ -449,12 +448,6 @@ elseif(CLANG AND DEPS AND NOT FREEBSD)
-   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -flto=thin")
- endif()
- 
--if(NOT IS_DIRECTORY "${CMAKE_SOURCE_DIR}/third-party/sqlite3")
--  WARNING_LOG("Cannot find git submodule third-party/sqlite3 directory")
--  WARNING_LOG("Please run: make deps or git submodule update --init")
--  message(FATAL_ERROR "No sqlite3 directory")
--endif()
--
- # Make sure deps were built before compiling (else show warning).
- execute_process(
-   COMMAND "${CMAKE_SOURCE_DIR}/tools/provision.sh" check "${CMAKE_BINARY_DIR}"
-@@ -521,6 +514,8 @@ endif()
- 
- if(APPLE)
-   LOG_PLATFORM("OS X")
-+elseif(OSQUERY_BUILD_PLATFORM STREQUAL "nixos")
-+  LOG_PLATFORM("NixOS")
- elseif(OSQUERY_BUILD_PLATFORM STREQUAL "debian")
-   LOG_PLATFORM("Debian")
- elseif(OSQUERY_BUILD_PLATFORM STREQUAL "ubuntu")
-@@ -570,7 +565,6 @@ if(POSIX AND DEPS)
-   endif()
- endif()
- 
--include_directories("${CMAKE_SOURCE_DIR}/third-party/sqlite3")
- include_directories("${CMAKE_SOURCE_DIR}/include")
- include_directories("${CMAKE_SOURCE_DIR}")
- 
-@@ -661,18 +655,6 @@ if(FREEBSD OR "${HAVE_TR1_TUPLE}" STREQUAL "")
-   add_definitions(-DGTEST_USE_OWN_TR1_TUPLE=0)
- endif()
- 
--set(GTEST_FLAGS
--  "-I${CMAKE_SOURCE_DIR}/third-party/googletest/googletest/include"
--  "-I${CMAKE_SOURCE_DIR}/third-party/googletest/googlemock/include"
--)
--join("${GTEST_FLAGS}" " " GTEST_FLAGS)
--
--if(NOT SKIP_TESTS)
--  add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/googletest")
--endif()
--
--add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/sqlite3")
--
- if(NOT FREEBSD)
-   add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/linenoise-ng")
- endif()
-diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt
-index e3842962..f405503d 100644
---- a/osquery/CMakeLists.txt
-+++ b/osquery/CMakeLists.txt
-@@ -33,8 +33,6 @@ if(CLANG AND POSIX)
-     -Wextra
-     -pedantic
-     -Wuseless-cast
--    -Wno-c99-extensions
--    -Wno-zero-length-array
-     -Wno-unused-parameter
-     -Wno-gnu-case-range
-     -Weffc++
-@@ -63,14 +61,7 @@ endif()
- 
- # Construct a set of all object files, starting with third-party and all
- # of the osquery core objects (sources from ADD_CORE_LIBRARY macros).
--if(FREEBSD)
--  set(OSQUERY_OBJECTS $<TARGET_OBJECTS:osquery_sqlite>)
--else()
--  set(OSQUERY_OBJECTS
--    $<TARGET_OBJECTS:osquery_sqlite>
--    $<TARGET_OBJECTS:linenoise-ng>
--  )
--endif()
-+set(OSQUERY_OBJECTS $<TARGET_OBJECTS:linenoise-ng>)
- 
- # Add subdirectories
- add_subdirectory(config)
-@@ -151,10 +142,11 @@ if(APPLE OR LINUX)
-   ADD_OSQUERY_LINK_ADDITIONAL("rocksdb_lite")
- elseif(FREEBSD)
-   ADD_OSQUERY_LINK_CORE("icuuc")
--  ADD_OSQUERY_LINK_CORE("linenoise")
-   ADD_OSQUERY_LINK_ADDITIONAL("rocksdb-lite")
- endif()
- 
-+ADD_OSQUERY_LINK_CORE("sqlite3")
-+
- if(POSIX)
-   ADD_OSQUERY_LINK_CORE("boost_system")
-   ADD_OSQUERY_LINK_CORE("boost_filesystem")
-@@ -172,10 +164,10 @@ endif()
- ADD_OSQUERY_LINK_CORE("glog${WO_KEY}")
- 
- if(POSIX)
--  # Hashing methods in core use libcrypto.
--  ADD_OSQUERY_LINK_CORE("crypto")
--
-+  ADD_OSQUERY_LINK_ADDITIONAL("benchmark")
-+  ADD_OSQUERY_LINK_ADDITIONAL("snappy")
-   ADD_OSQUERY_LINK_ADDITIONAL("ssl")
-+  ADD_OSQUERY_LINK_ADDITIONAL("crypto")
-   ADD_OSQUERY_LINK_ADDITIONAL("libpthread")
-   ADD_OSQUERY_LINK_ADDITIONAL("magic")
- endif()
-diff --git a/osquery/extensions/CMakeLists.txt b/osquery/extensions/CMakeLists.txt
-index 52f3bf80..066ed1c0 100644
---- a/osquery/extensions/CMakeLists.txt
-+++ b/osquery/extensions/CMakeLists.txt
-@@ -60,12 +60,6 @@ else()
-   )
- endif()
- 
--if(NOT WINDOWS)
--  add_compile_options(
--    -Wno-macro-redefined
--  )
--endif()
--
- ADD_OSQUERY_LIBRARY(TRUE osquery_extensions
-   ${THRIFT_GENERATED_FILES}
-   ${THRIFT_IMPL_FILE}
-diff --git a/osquery/logger/CMakeLists.txt b/osquery/logger/CMakeLists.txt
-index ab91bd24..d8364991 100644
---- a/osquery/logger/CMakeLists.txt
-+++ b/osquery/logger/CMakeLists.txt
-@@ -55,9 +55,9 @@ if(NOT SKIP_KAFKA AND NOT WINDOWS AND NOT FREEBSD)
- 
-   ADD_OSQUERY_LINK_ADDITIONAL("rdkafka")
- 
--  set(OSQUERY_LOGGER_KAFKA_PLUGINS_TESTS
--    "logger/plugins/tests/kafka_producer_tests.cpp"
--  )
-+  #set(OSQUERY_LOGGER_KAFKA_PLUGINS_TESTS
-+  #  "logger/plugins/tests/kafka_producer_tests.cpp"
-+  #)
- 
-   ADD_OSQUERY_TEST_ADDITIONAL(${OSQUERY_LOGGER_KAFKA_PLUGINS_TESTS})
- endif()
-diff --git a/osquery/tables/CMakeLists.txt b/osquery/tables/CMakeLists.txt
-index 3ecbb711..af7220d3 100644
---- a/osquery/tables/CMakeLists.txt
-+++ b/osquery/tables/CMakeLists.txt
-@@ -68,7 +68,7 @@ if(LINUX)
-   set(TABLE_PLATFORM "linux")
- 
-   ADD_OSQUERY_LINK_ADDITIONAL("libresolv.so")
--  ADD_OSQUERY_LINK_ADDITIONAL("cryptsetup devmapper lvm2app lvm-internal daemonclient")
-+  ADD_OSQUERY_LINK_ADDITIONAL("cryptsetup devmapper lvm2app")
-   ADD_OSQUERY_LINK_ADDITIONAL("gcrypt gpg-error")
-   ADD_OSQUERY_LINK_ADDITIONAL("blkid")
-   ADD_OSQUERY_LINK_ADDITIONAL("ip4tc")
-diff --git a/specs/windows/services.table b/specs/windows/services.table
-index 4ac24ee9..657d8b99 100644
---- a/specs/windows/services.table
-+++ b/specs/windows/services.table
-@@ -12,7 +12,7 @@ schema([
-     Column("path", TEXT, "Path to Service Executable"),
-     Column("module_path", TEXT, "Path to ServiceDll"),
-     Column("description", TEXT, "Service Description"),
--    Column("user_account", TEXT, "The name of the account that the service process will be logged on as when it runs. This name can be of the form Domain\UserName. If the account belongs to the built-in domain, the name can be of the form .\UserName."),
-+    Column("user_account", TEXT, "The name of the account that the service process will be logged on as when it runs. This name can be of the form Domain\\UserName. If the account belongs to the built-in domain, the name can be of the form .\\UserName."),
- ])
- implementation("system/windows/services@genServices")
- examples([