paperless-ng: init at 1.4.5

6 files changed, 541 insertions, 0 deletions

diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index 270e3070406..97d76e4984e 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -551,6 +551,7 @@
   ./services/misc/osrm.nix
   ./services/misc/packagekit.nix
   ./services/misc/paperless.nix
+  ./services/misc/paperless-ng.nix
   ./services/misc/parsoid.nix
   ./services/misc/plex.nix
   ./services/misc/plikd.nix
diff --git a/nixos/modules/services/misc/paperless-ng.nix b/nixos/modules/services/misc/paperless-ng.nix
new file mode 100644
index 00000000000..12d9a45d3a1
--- /dev/null
+++ b/nixos/modules/services/misc/paperless-ng.nix
@@ -0,0 +1,304 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+let
+  cfg = config.services.paperless-ng;
+
+  defaultUser = "paperless";
+
+  env = {
+    PAPERLESS_DATA_DIR = cfg.dataDir;
+    PAPERLESS_MEDIA_ROOT = cfg.mediaDir;
+    PAPERLESS_CONSUMPTION_DIR = cfg.consumptionDir;
+    GUNICORN_CMD_ARGS = "--bind=${cfg.address}:${toString cfg.port}";
+  } // lib.mapAttrs (_: toString) cfg.extraConfig;
+
+  manage = let
+    setupEnv = lib.concatStringsSep "\n" (mapAttrsToList (name: val: "export ${name}=\"${val}\"") env);
+  in pkgs.writeShellScript "manage" ''
+    ${setupEnv}
+    exec ${cfg.package}/bin/paperless-ng "$@"
+  '';
+
+  # Secure the services
+  defaultServiceConfig = {
+    TemporaryFileSystem = "/:ro";
+    BindReadOnlyPaths = [
+      "/nix/store"
+      "-/etc/resolv.conf"
+      "-/etc/nsswitch.conf"
+      "-/etc/hosts"
+      "-/etc/localtime"
+    ];
+    BindPaths = [
+      cfg.consumptionDir
+      cfg.dataDir
+      cfg.mediaDir
+    ];
+    CapabilityBoundingSet = "";
+    # ProtectClock adds DeviceAllow=char-rtc r
+    DeviceAllow = "";
+    LockPersonality = true;
+    MemoryDenyWriteExecute = true;
+    NoNewPrivileges = true;
+    PrivateDevices = true;
+    PrivateMounts = true;
+    # Needs to connect to redis
+    # PrivateNetwork = true;
+    PrivateTmp = true;
+    PrivateUsers = true;
+    ProcSubset = "pid";
+    ProtectClock = true;
+    # Breaks if the home dir of the user is in /home
+    # Also does not add much value in combination with the TemporaryFileSystem.
+    # ProtectHome = true;
+    ProtectHostname = true;
+    # Would re-mount paths ignored by temporary root
+    #ProtectSystem = "strict";
+    ProtectControlGroups = true;
+    ProtectKernelLogs = true;
+    ProtectKernelModules = true;
+    ProtectKernelTunables = true;
+    ProtectProc = "invisible";
+    RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
+    RestrictNamespaces = true;
+    RestrictRealtime = true;
+    RestrictSUIDSGID = true;
+    SystemCallArchitectures = "native";
+    SystemCallFilter = [ "@system-service" "~@privileged @resources @setuid @keyring" ];
+    # Does not work well with the temporary root
+    #UMask = "0066";
+  };
+in
+{
+  meta.maintainers = with maintainers; [ earvstedt Flakebi ];
+
+  options.services.paperless-ng = {
+    enable = mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Enable Paperless-ng.
+
+        When started, the Paperless database is automatically created if it doesn't
+        exist and updated if the Paperless package has changed.
+        Both tasks are achieved by running a Django migration.
+
+        A script to manage the Paperless instance (by wrapping Django's manage.py) is linked to
+        <literal>''${dataDir}/paperless-ng-manage</literal>.
+      '';
+    };
+
+    dataDir = mkOption {
+      type = types.str;
+      default = "/var/lib/paperless";
+      description = "Directory to store the Paperless data.";
+    };
+
+    mediaDir = mkOption {
+      type = types.str;
+      default = "${cfg.dataDir}/media";
+      defaultText = "\${dataDir}/consume";
+      description = "Directory to store the Paperless documents.";
+    };
+
+    consumptionDir = mkOption {
+      type = types.str;
+      default = "${cfg.dataDir}/consume";
+      defaultText = "\${dataDir}/consume";
+      description = "Directory from which new documents are imported.";
+    };
+
+    consumptionDirIsPublic = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Whether all users can write to the consumption dir.";
+    };
+
+    passwordFile = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      example = "/run/keys/paperless-ng-password";
+      description = ''
+        A file containing the superuser password.
+
+        A superuser is required to access the web interface.
+        If unset, you can create a superuser manually by running
+        <literal>''${dataDir}/paperless-ng-manage createsuperuser</literal>.
+
+        The default superuser name is <literal>admin</literal>. To change it, set
+        option <option>extraConfig.PAPERLESS_ADMIN_USER</option>.
+        WARNING: When changing the superuser name after the initial setup, the old superuser
+        will continue to exist.
+
+        To disable login for the web interface, set the following:
+        <literal>extraConfig.PAPERLESS_AUTO_LOGIN_USERNAME = "admin";</literal>.
+        WARNING: Only use this on a trusted system without internet access to Paperless.
+      '';
+    };
+
+    address = mkOption {
+      type = types.str;
+      default = "localhost";
+      description = "Web interface address.";
+    };
+
+    port = mkOption {
+      type = types.port;
+      default = 28981;
+      description = "Web interface port.";
+    };
+
+    extraConfig = mkOption {
+      type = types.attrs;
+      default = {};
+      description = ''
+        Extra paperless-ng config options.
+
+        See <link xlink:href="https://paperless-ng.readthedocs.io/en/latest/configuration.html">the documentation</link>
+        for available options.
+      '';
+      example = literalExample ''
+        {
+          PAPERLESS_OCR_LANGUAGE = "deu+eng";
+        }
+      '';
+    };
+
+    user = mkOption {
+      type = types.str;
+      default = defaultUser;
+      description = "User under which Paperless runs.";
+    };
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.paperless-ng;
+      defaultText = "pkgs.paperless-ng";
+      description = "The Paperless package to use.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = config.services.paperless.enable ->
+          (config.services.paperless.dataDir != cfg.dataDir && config.services.paperless.port != cfg.port);
+        message = "Paperless-ng replaces Paperless, either disable Paperless or assign a new dataDir and port to one of them";
+      }
+    ];
+
+    # Enable redis if no special url is set
+    services.redis.enable = mkIf (!hasAttr "PAPERLESS_REDIS" env) true;
+
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -"
+      "d '${cfg.mediaDir}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -"
+      (if cfg.consumptionDirIsPublic then
+        "d '${cfg.consumptionDir}' 777 - - - -"
+      else
+        "d '${cfg.consumptionDir}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -"
+      )
+    ];
+
+    systemd.services.paperless-ng-server = {
+      description = "Paperless document server";
+      serviceConfig = defaultServiceConfig // {
+        User = cfg.user;
+        ExecStart = "${cfg.package}/bin/paperless-ng qcluster";
+        Restart = "on-failure";
+      };
+      environment = env;
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "paperless-ng-consumer.service" "paperless-ng-web.service" ];
+
+      preStart = ''
+        ln -sf ${manage} ${cfg.dataDir}/paperless-ng-manage
+
+        # Auto-migrate on first run or if the package has changed
+        versionFile="${cfg.dataDir}/src-version"
+        if [[ $(cat "$versionFile" 2>/dev/null) != ${cfg.package} ]]; then
+          ${cfg.package}/bin/paperless-ng migrate
+          echo ${cfg.package} > "$versionFile"
+        fi
+      ''
+      + optionalString (cfg.passwordFile != null) ''
+        export PAPERLESS_ADMIN_USER="''${PAPERLESS_ADMIN_USER:-admin}"
+        export PAPERLESS_ADMIN_PASSWORD=$(cat "${cfg.dataDir}/superuser-password")
+        superuserState="$PAPERLESS_ADMIN_USER:$PAPERLESS_ADMIN_PASSWORD"
+        superuserStateFile="${cfg.dataDir}/superuser-state"
+
+        if [[ $(cat "$superuserStateFile" 2>/dev/null) != $superuserState ]]; then
+          ${cfg.package}/bin/paperless-ng manage_superuser
+          echo "$superuserState" > "$superuserStateFile"
+        fi
+      '';
+    };
+
+    # Password copying can't be implemented as a privileged preStart script
+    # in 'paperless-ng-server' because 'defaultServiceConfig' limits the filesystem
+    # paths accessible by the service.
+    systemd.services.paperless-ng-copy-password = mkIf (cfg.passwordFile != null) {
+      requiredBy = [ "paperless-ng-server.service" ];
+      before = [ "paperless-ng-server.service" ];
+      serviceConfig = {
+        ExecStart = ''
+          ${pkgs.coreutils}/bin/install --mode 600 --owner '${cfg.user}' --compare \
+            '${cfg.passwordFile}' '${cfg.dataDir}/superuser-password'
+        '';
+        Type = "oneshot";
+      };
+    };
+
+    systemd.services.paperless-ng-consumer = {
+      description = "Paperless document consumer";
+      serviceConfig = defaultServiceConfig // {
+        User = cfg.user;
+        ExecStart = "${cfg.package}/bin/paperless-ng document_consumer";
+        Restart = "on-failure";
+      };
+      environment = env;
+      # Bind to `paperless-ng-server` so that the consumer never runs
+      # during migrations
+      bindsTo = [ "paperless-ng-server.service" ];
+      after = [ "paperless-ng-server.service" ];
+    };
+
+    systemd.services.paperless-ng-web = {
+      description = "Paperless web server";
+      serviceConfig = defaultServiceConfig // {
+        User = cfg.user;
+        ExecStart = ''
+          ${pkgs.python3Packages.gunicorn}/bin/gunicorn \
+            -c ${cfg.package}/lib/paperless-ng/gunicorn.conf.py paperless.asgi:application
+        '';
+        Restart = "on-failure";
+
+        AmbientCapabilities = "CAP_NET_BIND_SERVICE";
+        CapabilityBoundingSet = "CAP_NET_BIND_SERVICE";
+        # gunicorn needs setuid
+        SystemCallFilter = defaultServiceConfig.SystemCallFilter ++ [ "@setuid" ];
+      };
+      environment = env // {
+        PATH = mkForce cfg.package.path;
+        PYTHONPATH = "${cfg.package.pythonPath}:${cfg.package}/lib/paperless-ng/src";
+      };
+      # Bind to `paperless-ng-server` so that the web server never runs
+      # during migrations
+      bindsTo = [ "paperless-ng-server.service" ];
+      after = [ "paperless-ng-server.service" ];
+    };
+
+    users = optionalAttrs (cfg.user == defaultUser) {
+      users.${defaultUser} = {
+        group = defaultUser;
+        uid = config.ids.uids.paperless;
+        home = cfg.dataDir;
+      };
+
+      groups.${defaultUser} = {
+        gid = config.ids.gids.paperless;
+      };
+    };
+  };
+}
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 8369f60e7b2..b41fc7a498d 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -335,6 +335,7 @@ in
   pam-u2f = handleTest ./pam-u2f.nix {};
   pantheon = handleTest ./pantheon.nix {};
   paperless = handleTest ./paperless.nix {};
+  paperless-ng = handleTest ./paperless-ng.nix {};
   pdns-recursor = handleTest ./pdns-recursor.nix {};
   peerflix = handleTest ./peerflix.nix {};
   pgjwt = handleTest ./pgjwt.nix {};
diff --git a/nixos/tests/paperless-ng.nix b/nixos/tests/paperless-ng.nix
new file mode 100644
index 00000000000..d8aafc2a08f
--- /dev/null
+++ b/nixos/tests/paperless-ng.nix
@@ -0,0 +1,36 @@
+import ./make-test-python.nix ({ lib, ... }: {
+  name = "paperless-ng";
+  meta.maintainers = with lib.maintainers; [ earvstedt Flakebi ];
+
+  nodes.machine = { pkgs, ... }: {
+    environment.systemPackages = with pkgs; [ imagemagick jq ];
+    services.paperless-ng = {
+      enable = true;
+      passwordFile = builtins.toFile "password" "admin";
+    };
+    virtualisation.memorySize = 1024;
+  };
+
+  testScript = ''
+    machine.wait_for_unit("paperless-ng-consumer.service")
+
+    with subtest("Create test doc"):
+        machine.succeed(
+            "convert -size 400x40 xc:white -font 'DejaVu-Sans' -pointsize 20 -fill black "
+            "-annotate +5+20 'hello world 16-10-2005' /var/lib/paperless/consume/doc.png"
+        )
+
+    with subtest("Web interface gets ready"):
+        machine.wait_for_unit("paperless-ng-web.service")
+        # Wait until server accepts connections
+        machine.wait_until_succeeds("curl -fs localhost:28981")
+
+    with subtest("Document is consumed"):
+        machine.wait_until_succeeds(
+            "(($(curl -u admin:admin -fs localhost:28981/api/documents/ | jq .count) == 1))"
+        )
+        assert "2005-10-16" in machine.succeed(
+            "curl -u admin:admin -fs localhost:28981/api/documents/ | jq '.results | .[0] | .created'"
+        )
+  '';
+})
diff --git a/pkgs/applications/office/paperless-ng/default.nix b/pkgs/applications/office/paperless-ng/default.nix
new file mode 100644
index 00000000000..66a548545b3
--- /dev/null
+++ b/pkgs/applications/office/paperless-ng/default.nix
@@ -0,0 +1,197 @@
+{ lib
+, fetchurl
+, nixosTests
+, python3
+, ghostscript
+, imagemagick
+, jbig2enc
+, ocrmypdf
+, optipng
+, pngquant
+, qpdf
+, tesseract4
+, unpaper
+, liberation_ttf
+}:
+
+let
+  py = python3.override {
+    packageOverrides = self: super: {
+      django = super.django_3;
+      django-picklefield = super.django-picklefield.overrideAttrs (oldAttrs: {
+        # Checks do not pass with django 3
+        doInstallCheck = false;
+      });
+      # Avoid warning in django-q versions > 1.3.4
+      # https://github.com/jonaswinkler/paperless-ng/issues/857
+      # https://github.com/Koed00/django-q/issues/526
+      django-q = super.django-q.overridePythonAttrs (oldAttrs: rec {
+        version = "1.3.4";
+        src = super.fetchPypi {
+          inherit (oldAttrs) pname;
+          inherit version;
+          sha256 = "Uj1U3PG2YVLBtlj5FPAO07UYo0MqnezUiYc4yo274Q8=";
+        };
+      });
+    };
+  };
+
+  path = lib.makeBinPath [ ghostscript imagemagick jbig2enc optipng pngquant qpdf tesseract4 unpaper ];
+in
+py.pkgs.pythonPackages.buildPythonApplication rec {
+  pname = "paperless-ng";
+  version = "1.4.5";
+
+  src = fetchurl {
+    url = "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-${version}/${pname}-${version}.tar.xz";
+    sha256 = "2PJb8j3oimlfiJ3gqjK6uTemzFdtAP2Mlm5RH09bx/E=";
+  };
+
+  format = "other";
+
+  # Make bind address configurable
+  # Fix tests with Pillow 8.3.1: https://github.com/jonaswinkler/paperless-ng/pull/1183
+  prePatch = ''
+    substituteInPlace gunicorn.conf.py --replace "bind = '0.0.0.0:8000'" ""
+    substituteInPlace src/paperless_tesseract/parsers.py --replace "return x" "return round(x)"
+  '';
+
+  propagatedBuildInputs = with py.pkgs.pythonPackages; [
+    aioredis
+    arrow
+    asgiref
+    async-timeout
+    attrs
+    autobahn
+    automat
+    blessed
+    certifi
+    cffi
+    channels-redis
+    channels
+    chardet
+    click
+    coloredlogs
+    concurrent-log-handler
+    constantly
+    cryptography
+    daphne
+    dateparser
+    django-cors-headers
+    django_extensions
+    django-filter
+    django-picklefield
+    django-q
+    django
+    djangorestframework
+    filelock
+    fuzzywuzzy
+    gunicorn
+    h11
+    hiredis
+    httptools
+    humanfriendly
+    hyperlink
+    idna
+    imap-tools
+    img2pdf
+    incremental
+    inotify-simple
+    inotifyrecursive
+    joblib
+    langdetect
+    lxml
+    msgpack
+    numpy
+    ocrmypdf
+    pathvalidate
+    pdfminer
+    pikepdf
+    pillow
+    pluggy
+    portalocker
+    psycopg2
+    pyasn1-modules
+    pyasn1
+    pycparser
+    pyopenssl
+    python-dateutil
+    python-dotenv
+    python-gnupg
+    python-Levenshtein
+    python_magic
+    pytz
+    pyyaml
+    redis
+    regex
+    reportlab
+    requests
+    scikit-learn
+    scipy
+    service-identity
+    six
+    sortedcontainers
+    sqlparse
+    threadpoolctl
+    tika
+    tqdm
+    twisted.extras.tls
+    txaio
+    tzlocal
+    urllib3
+    uvicorn
+    uvloop
+    watchdog
+    watchgod
+    wcwidth
+    websockets
+    whitenoise
+    whoosh
+    zope_interface
+  ];
+
+  doCheck = true;
+  checkInputs = with py.pkgs.pythonPackages; [
+    pytest
+    pytest-cov
+    pytest-django
+    pytest-env
+    pytest-sugar
+    pytest-xdist
+    factory_boy
+  ];
+
+  # The tests require:
+  # - PATH with runtime binaries
+  # - A temporary HOME directory for gnupg
+  # - XDG_DATA_DIRS with test-specific fonts
+  checkPhase = ''
+    pushd src
+    PATH="${path}:$PATH" HOME=$(mktemp -d) XDG_DATA_DIRS="${liberation_ttf}/share:$XDG_DATA_DIRS" pytest
+    popd
+  '';
+
+  installPhase = ''
+    mkdir -p $out/lib
+    cp -r . $out/lib/paperless-ng
+    chmod +x $out/lib/paperless-ng/src/manage.py
+    makeWrapper $out/lib/paperless-ng/src/manage.py $out/bin/paperless-ng \
+      --prefix PYTHONPATH : "$PYTHONPATH" \
+      --prefix PATH : "${path}"
+  '';
+
+  passthru = {
+    # PYTHONPATH of all dependencies used by the package
+    pythonPath = python3.pkgs.makePythonPath propagatedBuildInputs;
+    inherit path;
+
+    tests = { inherit (nixosTests) paperless-ng; };
+  };
+
+  meta = with lib; {
+    description = "A supercharged version of paperless: scan, index, and archive all of your physical documents";
+    homepage = "https://paperless-ng.readthedocs.io/en/latest/";
+    license = licenses.gpl3Only;
+    maintainers = with maintainers; [ earvstedt Flakebi ];
+  };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 9ece72bc0aa..af68bc489dc 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -7891,6 +7891,8 @@ with pkgs;
 
   paperless = callPackage ../applications/office/paperless { };
 
+  paperless-ng = callPackage ../applications/office/paperless-ng { };
+
   paperwork = callPackage ../applications/office/paperwork/paperwork-gtk.nix { };
 
   papertrail = callPackage ../tools/text/papertrail { };