Merge branch 'second-db' into 'master'

Use separate database connection for Celery results

See merge request !943

CHANGELOG.rst

@@ -15,6 +15,12 @@ Changed
 * [Dev] The (undocumented) setting PDF_CONTEXT_PROCESSORS is now named NON_REQUEST_CONTEXT_PROCESSORS
 * [Docker] Cache is now cleared if migrations are applied
 
+Fixed
+~~~~~
+
+* Celery progress could be inaccurate if recording progress during a transaction
+
+
 `2.7.1`_ - 2022-01-28
 ---------------------
 

aleksis/core/settings.py

@@ -12,6 +12,7 @@ from .util.core_helpers import get_app_packages, merge_app_settings, monkey_patc
 monkey_patch()
 
 IN_PYTEST = "PYTEST_CURRENT_TEST" in os.environ or "TOX_ENV_DIR" in os.environ
+PYTEST_SETUP_DATABASES = [("default", "default_oot")]
 
 ENVVAR_PREFIX_FOR_DYNACONF = "ALEKSIS"
 DIRS_FOR_DYNACONF = ["/etc/aleksis"]
@@ -227,6 +228,13 @@ DATABASES = {
     }
 }
 
+# Duplicate default database for out-of-transaction updates
+DATABASES["default_oot"] = DATABASES["default"].copy()
+DATABASE_ROUTERS = [
+    "aleksis.core.util.core_helpers.OOTRouter",
+]
+DATABASE_OOT_LABELS = ["django_celery_results"]
+
 merge_app_settings("DATABASES", DATABASES, False)
 
 REDIS_HOST = _settings.get("redis.host", "localhost")

aleksis/core/util/core_helpers.py

@@ -390,3 +390,54 @@ def create_default_celery_schedule():
         PeriodicTask.objects.create(
             name=f"{name} (default schedule)", task=name, **{attr: db_schedule}
         )
+
+
+class OOTRouter:
+    """Database router for operations that should run out of transaction.
+
+    This router routes database operations for certain apps through
+    the separate default_oot connection, to ensure that data get
+    updated immediately even during atomic transactions.
+    """
+
+    default_db = "default"
+    oot_db = "default_oot"
+
+    @property
+    def oot_labels(self):
+        return settings.DATABASE_OOT_LABELS
+
+    @property
+    def default_dbs(self):
+        return set((self.default_db, self.oot_db))
+
+    def is_same_db(self, db1: str, db2: str):
+        return set((db1, db2)).issubset(self.default_dbs)
+
+    def db_for_read(self, model: Model, **hints) -> Optional[str]:
+        if model._meta.app_label in self.oot_labels:
+            return self.oot_db
+
+        return None
+
+    def db_for_write(self, model: Model, **hints) -> Optional[str]:
+        return self.db_for_read(model, **hints)
+
+    def allow_relation(self, obj1: Model, obj2: Model, **hints) -> Optional[bool]:
+        # Allow relations between default database and OOT connection
+        # They are the same database
+        if self.is_same_db(obj1._state.db, obj2._state.db):
+            return True
+
+        return None
+
+    def allow_migrate(
+        self, db: str, app_label: str, model_name: Optional[str] = None, **hints
+    ) -> Optional[bool]:
+        # Never allow any migrations on the default_oot database
+        # It connects to the same database as default, so everything
+        # migrated there
+        if db == self.oot_db:
+            return False
+
+        return None