From b250fc121b785e972b8a2b92356db26905b37c45 Mon Sep 17 00:00:00 2001
From: Dominik George <dominik.george@teckids.org>
Date: Mon, 31 Jan 2022 12:32:01 +0100
Subject: [PATCH] Use separate database connection for Celery results
This ensures Celery progress is always updated and queried outside a
transaction.
---
CHANGELOG.rst | 6 ++++
aleksis/core/settings.py | 7 +++++
aleksis/core/util/core_helpers.py | 51 +++++++++++++++++++++++++++++++
3 files changed, 64 insertions(+)
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index c389ca8c9..9e27918ef 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -15,6 +15,12 @@ Changed
* [Dev] The (undocumented) setting PDF_CONTEXT_PROCESSORS is now named NON_REQUEST_CONTEXT_PROCESSORS
* [Docker] Cache is now cleared if migrations are applied
+Fixed
+~~~~~
+
+* Celery progress could be inaccurate if recording progress during a transaction
+
+
`2.7.1`_ - 2022-01-28
---------------------
diff --git a/aleksis/core/settings.py b/aleksis/core/settings.py
index b380f0dd8..6d1302e83 100644
--- a/aleksis/core/settings.py
+++ b/aleksis/core/settings.py
@@ -227,6 +227,13 @@ DATABASES = {
}
}
+# Duplicate default database for out-of-transaction updates
+DATABASES["default_oot"] = DATABASES["default"].copy()
+DATABASE_ROUTERS = [
+ "aleksis.core.util.core_helpers.OOTRouter",
+]
+DATABASE_OOT_LABELS = ["django_celery_results"]
+
merge_app_settings("DATABASES", DATABASES, False)
REDIS_HOST = _settings.get("redis.host", "localhost")
diff --git a/aleksis/core/util/core_helpers.py b/aleksis/core/util/core_helpers.py
index 55085d910..fc83b6050 100644
--- a/aleksis/core/util/core_helpers.py
+++ b/aleksis/core/util/core_helpers.py
@@ -390,3 +390,54 @@ def create_default_celery_schedule():
PeriodicTask.objects.create(
name=f"{name} (default schedule)", task=name, **{attr: db_schedule}
)
+
+
+class OOTRouter:
+ """Database router for operations that should run out of transaction.
+
+ This router routes database operations for certain apps through
+ the separate default_oot connection, to ensure that data get
+ updated immediately even during atomic transactions.
+ """
+
+ default_db = "default"
+ oot_db = "default_oot"
+
+ @property
+ def oot_labels(self):
+ return settings.DATABASE_OOT_LABELS
+
+ @property
+ def default_dbs(self):
+ return set((self.default_db, self.oot_db))
+
+ def is_same_db(self, db1: str, db2: str):
+ return set((db1, db2)).issubset(self.default_dbs)
+
+ def db_for_read(self, model: Model, **hints) -> Optional[str]:
+ if model._meta.app_label in self.oot_labels:
+ return self.oot_db
+
+ return None
+
+ def db_for_write(self, model: Model, **hints) -> Optional[str]:
+ return self.db_for_read(model, **hints)
+
+ def allow_relation(self, obj1: Model, obj2: Model, **hints) -> Optional[bool]:
+ # Allow relations between default database and OOT connection
+ # They are the same database
+ if self.is_same_db(obj1._state.db, obj2._state.db):
+ return True
+
+ return None
+
+ def allow_migrate(
+ self, db: str, app_label: str, model_name: Optional[str] = None, **hints
+ ) -> Optional[bool]:
+ # Never allow any migrations on the default_oot database
+ # It connects to the same database as default, so everything
+ # migrated there
+ if db == self.oot_db:
+ return False
+
+ return None
--
GitLab