sourcehut: fix Unix socket support for Redis

15 files changed, 47 insertions, 271 deletions

diff --git a/nixos/modules/services/misc/sourcehut/default.nix b/nixos/modules/services/misc/sourcehut/default.nix
index 70f6ea7ce97..f5e4742d902 100644
--- a/nixos/modules/services/misc/sourcehut/default.nix
+++ b/nixos/modules/services/misc/sourcehut/default.nix
@@ -83,7 +83,7 @@ let
   python = pkgs.sourcehut.python.withPackages (ps: with ps; [
     gunicorn
     eventlet
-    # For monitoring Celery: sudo -u listssrht celery --app listssrht.process -b redis+socket:///run/redis-sourcehut/redis.sock?virtual_host=5 flower
+    # For monitoring Celery: sudo -u listssrht celery --app listssrht.process -b redis+socket:///run/redis-sourcehut/redis.sock?virtual_host=1 flower
     flower
     # Sourcehut services
     srht
diff --git a/nixos/modules/services/misc/sourcehut/service.nix b/nixos/modules/services/misc/sourcehut/service.nix
index f1706ad0a6a..4ecc7a72669 100644
--- a/nixos/modules/services/misc/sourcehut/service.nix
+++ b/nixos/modules/services/misc/sourcehut/service.nix
@@ -148,7 +148,7 @@ in
     redis = {
       host = mkOption {
         type = types.str;
-        default = "unix:/run/redis-sourcehut-${srvsrht}/redis.sock?db=0";
+        default = "unix:///run/redis-sourcehut-${srvsrht}/redis.sock?db=0";
         example = "redis://shared.wireguard:6379/0";
         description = ''
           The redis host URL. This is used for caching and temporary storage, and must
diff --git a/nixos/tests/sourcehut.nix b/nixos/tests/sourcehut.nix
index 34a60247e00..6998f4f0c10 100644
--- a/nixos/tests/sourcehut.nix
+++ b/nixos/tests/sourcehut.nix
@@ -195,6 +195,7 @@ in
     # Testing metasrht
     machine.wait_for_unit("metasrht-api.service")
     machine.wait_for_unit("metasrht.service")
+    machine.wait_for_unit("metasrht-webhooks.service")
     machine.wait_for_open_port(5000)
     machine.succeed("curl -sL http://localhost:5000 | grep meta.${domain}")
     machine.succeed("curl -sL http://meta.${domain} | grep meta.${domain}")
diff --git a/pkgs/applications/version-management/sourcehut/builds.nix b/pkgs/applications/version-management/sourcehut/builds.nix
index bef1b175cd6..99d6a05c4eb 100644
--- a/pkgs/applications/version-management/sourcehut/builds.nix
+++ b/pkgs/applications/version-management/sourcehut/builds.nix
@@ -39,10 +39,6 @@ buildPythonPackage rec {
   inherit src version;
   pname = "buildsrht";
 
-  patches = [
-    # Revert change breaking Unix socket support for Redis
-    patches/redis-socket/build/0001-Revert-Add-build-submission-and-queue-monitoring.patch
-  ];
   postPatch = ''
     substituteInPlace Makefile \
       --replace "all: api worker" ""
diff --git a/pkgs/applications/version-management/sourcehut/core.nix b/pkgs/applications/version-management/sourcehut/core.nix
index 610d5e2647c..0648f21910f 100644
--- a/pkgs/applications/version-management/sourcehut/core.nix
+++ b/pkgs/applications/version-management/sourcehut/core.nix
@@ -48,6 +48,8 @@ buildPythonPackage rec {
   patches = [
     # Disable check for npm
     ./disable-npm-install.patch
+    # Fix Unix socket support in RedisQueueCollector
+    patches/redis-socket/core/0001-Fix-Unix-socket-support-in-RedisQueueCollector.patch
   ];
 
   nativeBuildInputs = [
diff --git a/pkgs/applications/version-management/sourcehut/git.nix b/pkgs/applications/version-management/sourcehut/git.nix
index e56022604fa..a721da0ecd3 100644
--- a/pkgs/applications/version-management/sourcehut/git.nix
+++ b/pkgs/applications/version-management/sourcehut/git.nix
@@ -58,10 +58,6 @@ buildPythonPackage rec {
   inherit src version;
   pname = "gitsrht";
 
-  patches = [
-    # Revert change breaking Unix socket support for Redis
-    patches/redis-socket/git/0001-Revert-Add-webhook-queue-monitoring.patch
-  ];
   postPatch = ''
     substituteInPlace Makefile \
       --replace "all: api gitsrht-dispatch gitsrht-keys gitsrht-shell gitsrht-update-hook" ""
diff --git a/pkgs/applications/version-management/sourcehut/lists.nix b/pkgs/applications/version-management/sourcehut/lists.nix
index 213ef3a8cfd..b9a00ffd213 100644
--- a/pkgs/applications/version-management/sourcehut/lists.nix
+++ b/pkgs/applications/version-management/sourcehut/lists.nix
@@ -30,10 +30,6 @@ buildPythonPackage rec {
     vendorSha256 = "sha256-xnmMkRSokbhWD+kz0XQ9AinYdm6/50FRBISURPvlzD0=";
   } // import ./fix-gqlgen-trimpath.nix {inherit unzip;});
 
-  patches = [
-    # Revert change breaking Unix socket support for Redis
-    patches/redis-socket/lists/0001-Revert-Add-webhook-queue-monitoring.patch
-  ];
   postPatch = ''
     substituteInPlace Makefile \
       --replace "all: api" ""
diff --git a/pkgs/applications/version-management/sourcehut/meta.nix b/pkgs/applications/version-management/sourcehut/meta.nix
index 79293736e4a..896279addd3 100644
--- a/pkgs/applications/version-management/sourcehut/meta.nix
+++ b/pkgs/applications/version-management/sourcehut/meta.nix
@@ -40,10 +40,6 @@ buildPythonPackage rec {
   pname = "metasrht";
   inherit version src;
 
-  patches = [
-    # Revert change breaking Unix socket support for Redis
-    patches/redis-socket/meta/0001-Revert-Add-webhook-queue-monitoring.patch
-  ];
   postPatch = ''
     substituteInPlace Makefile \
       --replace "all: api" ""
diff --git a/pkgs/applications/version-management/sourcehut/patches/redis-socket/build/0001-Revert-Add-build-submission-and-queue-monitoring.patch b/pkgs/applications/version-management/sourcehut/patches/redis-socket/build/0001-Revert-Add-build-submission-and-queue-monitoring.patch
deleted file mode 100644
index ae8e95ddc0c..00000000000
--- a/pkgs/applications/version-management/sourcehut/patches/redis-socket/build/0001-Revert-Add-build-submission-and-queue-monitoring.patch
+++ /dev/null
@@ -1,69 +0,0 @@
-From 069b03f85847ed4a9223183b62ee53f420838911 Mon Sep 17 00:00:00 2001
-From: Julien Moutinho <julm+srht@sourcephile.fr>
-Date: Thu, 16 Dec 2021 04:54:24 +0100
-Subject: [PATCH builds.sr.ht] Revert "Add build submission and queue
- monitoring"
-
-This reverts commit 690f1aa16c77e418dc40109cd5e8fdf4a7ed947a.
-
-This has broken Unix socket support for Redis
-See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E
----
- buildsrht/app.py    | 3 ---
- buildsrht/runner.py | 9 +--------
- 2 files changed, 1 insertion(+), 11 deletions(-)
-
-diff --git a/buildsrht/app.py b/buildsrht/app.py
-index e5321a2..7c9977c 100644
---- a/buildsrht/app.py
-+++ b/buildsrht/app.py
-@@ -36,9 +36,6 @@ class BuildApp(SrhtFlask):
-         self.register_blueprint(secrets)
-         self.register_blueprint(gql_blueprint)
- 
--        from buildsrht.runner import builds_queue_metrics_collector
--        self.metrics_registry.register(builds_queue_metrics_collector)
--
-         @self.context_processor
-         def inject():
-             return {
-diff --git a/buildsrht/runner.py b/buildsrht/runner.py
-index 7773452..0389c8e 100644
---- a/buildsrht/runner.py
-+++ b/buildsrht/runner.py
-@@ -5,13 +5,10 @@ from srht.config import cfg
- from srht.database import db
- from srht.email import send_email
- from srht.oauth import UserType
--from srht.metrics import RedisQueueCollector
--from prometheus_client import Counter
- 
- allow_free = cfg("builds.sr.ht", "allow-free", default="no") == "yes"
- 
--builds_broker = cfg("builds.sr.ht", "redis")
--runner = Celery('builds', broker=builds_broker, config_source={
-+runner = Celery('builds', broker=cfg("builds.sr.ht", "redis"), config_source={
-     "CELERY_TASK_SERIALIZER": "json",
-     "CELERY_ACCEPT_CONTENT": ["json"],
-     "CELERY_RESULT_SERIALIZER": "json",
-@@ -19,9 +16,6 @@ runner = Celery('builds', broker=builds_broker, config_source={
-     "CELERY_TASK_PROTOCOL": 1
- })
- 
--builds_queue_metrics_collector = RedisQueueCollector(builds_broker, "buildsrht_builds", "Number of builds currently in queue")
--builds_submitted = Counter("buildsrht_builds_submited", "Number of builds submitted")
--
- def queue_build(job, manifest):
-     from buildsrht.types import JobStatus
-     job.status = JobStatus.queued
-@@ -34,7 +28,6 @@ def queue_build(job, manifest):
-                 cfg("sr.ht", "owner-email"),
-                 "Cryptocurrency mining attempt on builds.sr.ht")
-     else:
--        builds_submitted.inc()
-         run_build.delay(job.id, manifest.to_dict())
- 
- def requires_payment(user):
--- 
-2.34.0
-
diff --git a/pkgs/applications/version-management/sourcehut/patches/redis-socket/core/0001-Fix-Unix-socket-support-in-RedisQueueCollector.patch b/pkgs/applications/version-management/sourcehut/patches/redis-socket/core/0001-Fix-Unix-socket-support-in-RedisQueueCollector.patch
new file mode 100644
index 00000000000..889331f3e07
--- /dev/null
+++ b/pkgs/applications/version-management/sourcehut/patches/redis-socket/core/0001-Fix-Unix-socket-support-in-RedisQueueCollector.patch
@@ -0,0 +1,42 @@
+From 76dd636151735671be74ba9d55f773e190e22827 Mon Sep 17 00:00:00 2001
+From: Julien Moutinho <julm+srht@sourcephile.fr>
+Date: Fri, 13 May 2022 22:40:46 +0200
+Subject: [PATCH core.sr.ht] Fix Unix socket support in RedisQueueCollector
+
+The broker URL is not necessarily in the format expected by Redis.from_url
+
+Especially, Redis.from_url supports this format for Unix sockets:
+    unix:///run/redis-sourcehut-metasrht/redis.sock?db=0
+See https://redis-py.readthedocs.io/en/stable/#redis.ConnectionPool.from_url
+
+Whereas Celery+Kombu support Redis but also other transports
+and thus expect another scheme:
+    redis+socket:///run/redis-sourcehut-metasrht/redis.sock?virtual_host=1
+See https://docs.celeryproject.org/en/stable/userguide/configuration.html#redis-backend-settings
+and https://github.com/celery/celery/blob/e5d99801e4b56a02af4a2e183879c767228d2817/celery/backends/redis.py#L299-L352
+and https://github.com/celery/kombu/blob/master/kombu/utils/url.py
+---
+ srht/metrics.py | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/srht/metrics.py b/srht/metrics.py
+index 68caf8e..2df5777 100644
+--- a/srht/metrics.py
++++ b/srht/metrics.py
+@@ -1,11 +1,12 @@
+ import time
++from celery import Celery
+ from prometheus_client.metrics_core import GaugeMetricFamily
+ from redis import Redis, ResponseError
+ 
+ 
+ class RedisQueueCollector:
+     def __init__(self, broker, name, documentation, queue_name="celery"):
+-        self.redis = Redis.from_url(broker)
++        self.redis = Celery("collector", broker=broker).connection_for_read().channel().client
+         self.queue_name = queue_name
+         self.name = name
+         self.documentation = documentation
+-- 
+2.35.1
+
diff --git a/pkgs/applications/version-management/sourcehut/patches/redis-socket/git/0001-Revert-Add-webhook-queue-monitoring.patch b/pkgs/applications/version-management/sourcehut/patches/redis-socket/git/0001-Revert-Add-webhook-queue-monitoring.patch
deleted file mode 100644
index f9235102b95..00000000000
--- a/pkgs/applications/version-management/sourcehut/patches/redis-socket/git/0001-Revert-Add-webhook-queue-monitoring.patch
+++ /dev/null
@@ -1,33 +0,0 @@
-diff --git a/gitsrht/app.py b/gitsrht/app.py
-index eaae395..731bb03 100644
---- a/gitsrht/app.py
-+++ b/gitsrht/app.py
-@@ -55,9 +55,6 @@ class GitApp(SrhtFlask):
-         self.add_template_filter(url_quote)
-         self.add_template_filter(commit_links)
- 
--        from gitsrht.webhooks import webhook_metrics_collector
--        self.metrics_registry.register(webhook_metrics_collector)
--
-         @self.context_processor
-         def inject():
-             notice = session.get("notice")
-diff --git a/gitsrht/webhooks.py b/gitsrht/webhooks.py
-index 36486d0..0109428 100644
---- a/gitsrht/webhooks.py
-+++ b/gitsrht/webhooks.py
-@@ -7,12 +7,10 @@ if not hasattr(db, "session"):
-     db.init()
- from srht.webhook import Event
- from srht.webhook.celery import CeleryWebhook, make_worker
--from srht.metrics import RedisQueueCollector
-+from scmsrht.webhooks import UserWebhook
- import sqlalchemy as sa
- 
--webhook_broker = cfg("git.sr.ht", "webhooks")
--worker = make_worker(broker=webhook_broker)
--webhook_metrics_collector = RedisQueueCollector(webhook_broker, "srht_webhooks", "Webhook queue length")
-+worker = make_worker(broker=cfg("git.sr.ht", "webhooks"))
- 
- class UserWebhook(CeleryWebhook):
-     events = [
diff --git a/pkgs/applications/version-management/sourcehut/patches/redis-socket/lists/0001-Revert-Add-webhook-queue-monitoring.patch b/pkgs/applications/version-management/sourcehut/patches/redis-socket/lists/0001-Revert-Add-webhook-queue-monitoring.patch
deleted file mode 100644
index 872d285b9bb..00000000000
--- a/pkgs/applications/version-management/sourcehut/patches/redis-socket/lists/0001-Revert-Add-webhook-queue-monitoring.patch
+++ /dev/null
@@ -1,48 +0,0 @@
-From 730e090f31b150d42be4b4722751f8e4610835b0 Mon Sep 17 00:00:00 2001
-From: Julien Moutinho <julm+srht@sourcephile.fr>
-Date: Sat, 12 Feb 2022 00:38:12 +0100
-Subject: [PATCH lists.sr.ht] Revert "Add webhook queue monitoring"
-
-This reverts commit e74e344808e8d523a9786cefcbf64c9a247d7a0e.
-
-Which has broken Unix socket support for Redis.
-See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E
----
- listssrht/app.py      | 3 ---
- listssrht/webhooks.py | 5 +----
- 2 files changed, 1 insertion(+), 7 deletions(-)
-
-diff --git a/listssrht/app.py b/listssrht/app.py
-index aec59f3..83a355d 100644
---- a/listssrht/app.py
-+++ b/listssrht/app.py
-@@ -29,9 +29,6 @@ class ListsApp(SrhtFlask):
-         self.register_blueprint(user)
-         self.register_blueprint(gql_blueprint)
- 
--        from listssrht.webhooks import webhook_metrics_collector
--        self.metrics_registry.register(webhook_metrics_collector)
--
-         @self.context_processor
-         def inject():
-             from listssrht.types import ListAccess
-diff --git a/listssrht/webhooks.py b/listssrht/webhooks.py
-index ae5b1cb..86421ba 100644
---- a/listssrht/webhooks.py
-+++ b/listssrht/webhooks.py
-@@ -8,11 +8,8 @@ if not hasattr(db, "session"):
-     db.init()
- from srht.webhook import Event
- from srht.webhook.celery import CeleryWebhook, make_worker
--from srht.metrics import RedisQueueCollector
- 
--webhook_broker = cfg("lists.sr.ht", "webhooks")
--worker = make_worker(broker=webhook_broker)
--webhook_metrics_collector = RedisQueueCollector(webhook_broker, "srht_webhooks", "Webhook queue length")
-+worker = make_worker(broker=cfg("lists.sr.ht", "webhooks"))
- 
- class ListWebhook(CeleryWebhook):
-     events = [
--- 
-2.34.1
-
diff --git a/pkgs/applications/version-management/sourcehut/patches/redis-socket/meta/0001-Revert-Add-webhook-queue-monitoring.patch b/pkgs/applications/version-management/sourcehut/patches/redis-socket/meta/0001-Revert-Add-webhook-queue-monitoring.patch
deleted file mode 100644
index 9ec37670b06..00000000000
--- a/pkgs/applications/version-management/sourcehut/patches/redis-socket/meta/0001-Revert-Add-webhook-queue-monitoring.patch
+++ /dev/null
@@ -1,48 +0,0 @@
-From d88bee195797c6c294320617ff14798da94cd0f3 Mon Sep 17 00:00:00 2001
-From: Julien Moutinho <julm+srht@sourcephile.fr>
-Date: Thu, 16 Dec 2021 04:52:08 +0100
-Subject: [PATCH meta.sr.ht] Revert "Add webhook queue monitoring"
-
-This reverts commit 9931df3c23094af5179df9ef019ca732b8125dac.
-
-This has broken Unix socket support for Redis.
-See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E
----
- metasrht/app.py      | 3 ---
- metasrht/webhooks.py | 5 +----
- 2 files changed, 1 insertion(+), 7 deletions(-)
-
-diff --git a/metasrht/app.py b/metasrht/app.py
-index b190875..89c59bc 100644
---- a/metasrht/app.py
-+++ b/metasrht/app.py
-@@ -49,9 +49,6 @@ class MetaApp(SrhtFlask):
-             from metasrht.blueprints.billing import billing
-             self.register_blueprint(billing)
- 
--        from metasrht.webhooks import webhook_metrics_collector
--        self.metrics_registry.register(webhook_metrics_collector)
--
-         @self.context_processor
-         def inject():
-             return {
-diff --git a/metasrht/webhooks.py b/metasrht/webhooks.py
-index 3e1149e..3f0ba01 100644
---- a/metasrht/webhooks.py
-+++ b/metasrht/webhooks.py
-@@ -7,11 +7,8 @@ if not hasattr(db, "session"):
-     db.init()
- from srht.webhook import Event
- from srht.webhook.celery import CeleryWebhook, make_worker
--from srht.metrics import RedisQueueCollector
- 
--webhook_broker = cfg("meta.sr.ht", "webhooks", "redis://")
--worker = make_worker(broker=webhook_broker)
--webhook_metrics_collector = RedisQueueCollector(webhook_broker, "srht_webhooks", "Webhook queue length")
-+worker = make_worker(broker=cfg("meta.sr.ht", "webhooks", "redis://"))
- 
- class UserWebhook(CeleryWebhook):
-     events = [
--- 
-2.34.0
-
diff --git a/pkgs/applications/version-management/sourcehut/patches/redis-socket/todo/0001-Revert-Add-webhook-queue-monitoring.patch b/pkgs/applications/version-management/sourcehut/patches/redis-socket/todo/0001-Revert-Add-webhook-queue-monitoring.patch
deleted file mode 100644
index 861608c1184..00000000000
--- a/pkgs/applications/version-management/sourcehut/patches/redis-socket/todo/0001-Revert-Add-webhook-queue-monitoring.patch
+++ /dev/null
@@ -1,50 +0,0 @@
-From 42a27ea60d8454552d54e1f51f1b976d1067fc32 Mon Sep 17 00:00:00 2001
-From: Julien Moutinho <julm+srht@sourcephile.fr>
-Date: Sat, 12 Feb 2022 00:30:29 +0100
-Subject: [PATCH todo.sr.ht] Revert "Add webhook queue monitoring"
-
-This reverts commit 320a5e8f7cd16ca43928c36f0320593f84d986fa.
-
-Which has broken Unix socket support for Redis.
-See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E
----
- todosrht/flask.py    | 3 ---
- todosrht/webhooks.py | 6 +-----
- 2 files changed, 1 insertion(+), 8 deletions(-)
-
-diff --git a/todosrht/flask.py b/todosrht/flask.py
-index 5e8ac66..9d0fd27 100644
---- a/todosrht/flask.py
-+++ b/todosrht/flask.py
-@@ -43,9 +43,6 @@ class TodoApp(SrhtFlask):
-         self.add_template_filter(urls.tracker_url)
-         self.add_template_filter(urls.user_url)
- 
--        from todosrht.webhooks import webhook_metrics_collector
--        self.metrics_registry.register(webhook_metrics_collector)
--
-         @self.context_processor
-         def inject():
-             return {
-diff --git a/todosrht/webhooks.py b/todosrht/webhooks.py
-index eb8e08a..950047f 100644
---- a/todosrht/webhooks.py
-+++ b/todosrht/webhooks.py
-@@ -7,13 +7,9 @@ if not hasattr(db, "session"):
-     db.init()
- from srht.webhook import Event
- from srht.webhook.celery import CeleryWebhook, make_worker
--from srht.metrics import RedisQueueCollector
- import sqlalchemy as sa
- 
--
--webhooks_broker = cfg("todo.sr.ht", "webhooks")
--worker = make_worker(broker=webhooks_broker)
--webhook_metrics_collector = RedisQueueCollector(webhooks_broker, "srht_webhooks", "Webhook queue length")
-+worker = make_worker(broker=cfg("todo.sr.ht", "webhooks"))
- 
- import todosrht.tracker_import
- 
--- 
-2.34.1
-
diff --git a/pkgs/applications/version-management/sourcehut/todo.nix b/pkgs/applications/version-management/sourcehut/todo.nix
index 9dbe86e6f63..67367b71ff9 100644
--- a/pkgs/applications/version-management/sourcehut/todo.nix
+++ b/pkgs/applications/version-management/sourcehut/todo.nix
@@ -23,16 +23,11 @@ buildPythonPackage rec {
     sha256 = "sha256-m7FY+jXpwPnK1+b1iQiDGe8JPfAFQp65BzGH6WvNwhM=";
   };
 
-  patches = [
-    # Revert change breaking Unix socket support for Redis
-    patches/redis-socket/todo/0001-Revert-Add-webhook-queue-monitoring.patch
-  ];
   postPatch = ''
     substituteInPlace Makefile \
       --replace "all: api" ""
   '';
 
-
   todosrht-api = buildGoModule ({
     inherit src version;
     pname = "todosrht-api";