Merge branch 'master' into 332-allow-automatic-linking-of-persons-to-account-by-e-mail-address

.gitlab-ci.yml

@@ -2,8 +2,14 @@ include:
     - project: "AlekSIS/official/AlekSIS"
       file: /ci/general.yml
     - project: "AlekSIS/official/AlekSIS"
-      file: /ci/test.yml
+      file: /ci/test/test.yml
     - project: "AlekSIS/official/AlekSIS"
-      file: /ci/build_dist.yml
+      file: /ci/test/lint.yml
     - project: "AlekSIS/official/AlekSIS"
-      file: /ci/deploy_pypi.yml
+      file: /ci/test/security.yml
+    - project: "AlekSIS/official/AlekSIS"
+      file: /ci/build/dist.yml
+    - project: "AlekSIS/official/AlekSIS"
+      file: /ci/deploy/pages.yml
+    - project: "AlekSIS/official/AlekSIS"
+      file: /ci/deploy/pypi.yml

aleksis/core/admin.py

@@ -10,6 +10,7 @@ from .models import (
     Announcement,
     AnnouncementRecipient,
     CustomMenuItem,
+    DataCheckResult,
     Group,
     Notification,
     Person,
@@ -33,3 +34,4 @@ class AnnouncementAdmin(BaseModelAdmin, VersionAdmin):
 
 
 admin.site.register(Announcement, AnnouncementAdmin)
+admin.site.register(DataCheckResult)

aleksis/core/apps.py

 from typing import Any, List, Optional, Tuple
 
 import django.apps
+from django.apps import apps
 from django.conf import settings
 from django.db import OperationalError, ProgrammingError
 from django.http import HttpRequest
 from django.utils.module_loading import autodiscover_modules
 
 from dynamic_preferences.registries import preference_models
+from health_check.plugins import plugin_dir
 
 from .registries import (
     group_preferences_registry,
@@ -52,6 +54,22 @@ class CoreConfig(AppConfig):
 
         self._refresh_authentication_backends()
 
+        self._load_data_checks()
+
+        from .health_checks import DataChecksHealthCheckBackend
+
+        plugin_dir.register(DataChecksHealthCheckBackend)
+
+    @classmethod
+    def _load_data_checks(cls):
+        """Get all data checks from all loaded models."""
+        from aleksis.core.data_checks import DataCheckRegistry
+
+        data_checks = []
+        for model in apps.get_models():
+            data_checks += getattr(model, "data_checks", [])
+        DataCheckRegistry.data_checks = data_checks
+
     @classmethod
     def _refresh_authentication_backends(cls):
         """Refresh config list of enabled authentication backends."""

aleksis/core/data_checks.py

+import logging
+
+from django.contrib.contenttypes.models import ContentType
+from django.db.models.aggregates import Count
+from django.utils.functional import classproperty
+from django.utils.translation import gettext as _
+
+import reversion
+from reversion import set_comment
+from templated_email import send_templated_mail
+
+from .util.core_helpers import celery_optional, get_site_preferences
+
+
+class SolveOption:
+    """Define a solve option for one or more data checks.
+
+    Solve options are used in order to give the data admin typical
+    solutions to a data issue detected by a data check.
+
+    Example definition
+
+    .. code-block:: python
+
+        from aleksis.core.data_checks import SolveOption
+        from django.utils.translation import gettext as _
+
+        class DeleteSolveOption(SolveOption):
+            name = "delete" # has to be unqiue
+            verbose_name = _("Delete") # should make use of i18n
+
+            @classmethod
+            def solve(cls, check_result: "DataCheckResult"):
+                check_result.related_object.delete()
+                check_result.delete()
+
+    After the solve option has been successfully executed,
+    the corresponding data check result has to be deleted.
+    """
+
+    name: str = "default"
+    verbose_name: str = ""
+
+    @classmethod
+    def solve(cls, check_result: "DataCheckResult"):
+        pass
+
+
+class IgnoreSolveOption(SolveOption):
+    """Mark the object with data issues as solved."""
+
+    name = "ignore"
+    verbose_name = _("Ignore problem")
+
+    @classmethod
+    def solve(cls, check_result: "DataCheckResult"):
+        """Mark the object as solved without doing anything more."""
+        check_result.solved = True
+        check_result.save()
+
+
+class DataCheck:
+    """Define a data check.
+
+    Data checks should be used to search objects of
+    any type which are broken or need some extra action.
+
+    Defining data checks
+    --------------------
+    Data checks are defined by inheriting from the class DataCheck
+    and registering the inherited class in the data check registry.
+
+    Example:
+
+    ``data_checks.py``
+    ******************
+
+    .. code-block:: python
+
+        from aleksis.core.data_checks import DataCheck, DATA_CHECK_REGISTRY
+        from django.utils.translation import gettext as _
+
+        class ExampleDataCheck(DataCheck):
+            name = "example" # has to be unique
+            verbose_name = _("Ensure that there are no examples.")
+            problem_name = _("There is an example.") # should both make use of i18n
+
+            solve_options = {
+                IgnoreSolveOption.name: IgnoreSolveOption
+            }
+
+            @classmethod
+            def check_data(cls):
+                from example_app.models import ExampleModel
+
+                wrong_examples = ExampleModel.objects.filter(wrong_value=True)
+
+                for example in wrong_examples:
+                    cls.register_result(example)
+
+    ``models.py``
+    *************
+
+    .. code-block:: python
+
+        from .data_checks import ExampleDataCheck
+
+        # ...
+
+        class ExampleModel(Model):
+            data_checks = [ExampleDataCheck]
+
+
+    Solve options are used in order to give the data admin typical solutions to this specific issue.
+    They are defined by inheriting from SolveOption.
+    More information about defining solve options can be find there.
+
+    The dictionary ``solve_options`` should include at least the IgnoreSolveOption,
+    but preferably also own solve options. The keys in this dictionary
+    have to be ``<YourOption>SolveOption.name``
+    and the values must be the corresponding solve option classes.
+
+    The class method ``check_data`` does the actual work. In this method
+    your code should find all objects with issues and should register
+    them in the result database using the class method ``register_result``.
+
+    Data checks have to be registered in their corresponding model.
+    This can be done by adding a list ``data_checks``
+    containing the data check classes.
+
+    Executing data checks
+    ---------------------
+    The data checks can be executed by using the
+    celery task named ``aleksis.core.data_checks.check_data``.
+    We recommend to create a periodic task in the backend
+    which executes ``check_data`` on a regular base (e. g. every day).
+
+    .. warning::
+        To use the option described above, you must have setup celery properly.
+
+    Notifications about results
+    ---------------------------
+    The data check tasks can notify persons via email
+    if there are new data issues. You can set these persons
+    by adding them to the preference
+    ``Email recipients for data checks problem emails`` in the site configuration.
+
+    To enable this feature, you also have to activate
+    the preference ``Send emails if data checks detect problems``.
+    """  # noqa: D412
+
+    name: str = ""
+    verbose_name: str = ""
+    problem_name: str = ""
+
+    solve_options = {IgnoreSolveOption.name: IgnoreSolveOption}
+
+    @classmethod
+    def check_data(cls):
+        """Find all objects with data issues and register them."""
+        pass
+
+    @classmethod
+    def solve(cls, check_result: "DataCheckResult", solve_option: str):
+        """Execute a solve option for an object detected by this check.
+
+        :param check_result: The result item from database
+        :param solve_option: The name of the solve option that should be executed
+        """
+        with reversion.create_revision():
+            solve_option_obj = cls.solve_options[solve_option]
+            set_comment(
+                _(
+                    f"Solve option '{solve_option_obj.verbose_name}' "
+                    f"for data check '{cls.verbose_name}'"
+                )
+            )
+            solve_option_obj.solve(check_result)
+
+    @classmethod
+    def register_result(cls, instance) -> "DataCheckResult":
+        """Register an object with data issues in the result database.
+
+        :param instance: The affected object
+        :return: The database entry
+        """
+        from aleksis.core.models import DataCheckResult
+
+        ct = ContentType.objects.get_for_model(instance)
+        result = DataCheckResult.objects.get_or_create(
+            check=cls.name, content_type=ct, object_id=instance.id
+        )
+        return result
+
+
+class DataCheckRegistry:
+    """Create central registry for all data checks in AlekSIS."""
+
+    data_checks = []
+
+    @classproperty
+    def data_checks_by_name(cls):
+        return {check.name: check for check in cls.data_checks}
+
+    @classproperty
+    def data_checks_choices(cls):
+        return [(check.name, check.verbose_name) for check in cls.data_checks]
+
+
+@celery_optional
+def check_data():
+    """Execute all registered data checks and send email if activated."""
+    for check in DataCheckRegistry.data_checks:
+        logging.info(f"Run check: {check.verbose_name}")
+        check.check_data()
+
+    if get_site_preferences()["general__data_checks_send_emails"]:
+        send_emails_for_data_checks()
+
+
+def send_emails_for_data_checks():
+    """Notify one or more recipients about new problems with data.
+
+    Recipients can be set in dynamic preferences.
+    """
+    from .models import DataCheckResult  # noqa
+
+    results = DataCheckResult.objects.filter(solved=False, sent=False)
+
+    if results.exists():
+        results_by_check = results.values("check").annotate(count=Count("check"))
+
+        results_with_checks = []
+        for result in results_by_check:
+            results_with_checks.append(
+                (DataCheckRegistry.data_checks_by_name[result["check"]], result["count"])
+            )
+
+        recipient_list = [
+            p.mail_sender
+            for p in get_site_preferences()["general__data_checks_recipients"]
+            if p.email
+        ]
+
+        for group in get_site_preferences()["general__data_checks_recipient_groups"]:
+            recipient_list += [p.mail_sender for p in group.announcement_recipients if p.email]
+
+        send_templated_mail(
+            template_name="data_checks",
+            from_email=get_site_preferences()["mail__address"],
+            recipient_list=recipient_list,
+            context={"results": results_with_checks},
+        )
+
+        logging.info("Sent notification email because of unsent data checks")
+
+        results.update(sent=True)

aleksis/core/health_checks.py

+from django.utils.translation import gettext as _
+
+from health_check.backends import BaseHealthCheckBackend
+
+from aleksis.core.models import DataCheckResult
+
+
+class DataChecksHealthCheckBackend(BaseHealthCheckBackend):
+    """Checks whether there are unresolved data problems."""
+
+    critical_service = False
+
+    def check_status(self):
+        if DataCheckResult.objects.filter(solved=False).exists():
+            self.add_error(_("There are unresolved data problems."))
+
+    def identifier(self):
+        return self.__class__.__name__

aleksis/core/locale/ar/LC_MESSAGES/djangojs.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-08-02 16:29+0200\n"
+"POT-Creation-Date: 2021-01-11 21:30+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -18,14 +18,18 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=6; plural=n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 && n%100<=99 ? 4 : 5;\n"
 
-#: static/js/main.js:21
+#: static/js/main.js:15
 msgid "Today"
 msgstr ""
 
-#: static/js/main.js:22
+#: static/js/main.js:16
 msgid "Cancel"
 msgstr ""
 
-#: static/js/main.js:23
+#: static/js/main.js:17
 msgid "OK"
 msgstr ""
+
+#: static/js/main.js:118
+msgid "This page may contain outdated information since there is no internet connection."
+msgstr ""

aleksis/core/locale/de_DE/LC_MESSAGES/djangojs.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-08-02 16:29+0200\n"
+"POT-Creation-Date: 2021-01-11 21:30+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -17,14 +17,18 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#: static/js/main.js:21
+#: static/js/main.js:15
 msgid "Today"
 msgstr ""
 
-#: static/js/main.js:22
+#: static/js/main.js:16
 msgid "Cancel"
 msgstr ""
 
-#: static/js/main.js:23
+#: static/js/main.js:17
 msgid "OK"
 msgstr ""
+
+#: static/js/main.js:118
+msgid "This page may contain outdated information since there is no internet connection."
+msgstr ""

aleksis/core/locale/fr/LC_MESSAGES/djangojs.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-08-02 16:29+0200\n"
+"POT-Creation-Date: 2021-01-11 21:30+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -18,14 +18,18 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n > 1);\n"
 
-#: static/js/main.js:21
+#: static/js/main.js:15
 msgid "Today"
 msgstr ""
 
-#: static/js/main.js:22
+#: static/js/main.js:16
 msgid "Cancel"
 msgstr ""
 
-#: static/js/main.js:23
+#: static/js/main.js:17
 msgid "OK"
 msgstr ""
+
+#: static/js/main.js:118
+msgid "This page may contain outdated information since there is no internet connection."
+msgstr ""

aleksis/core/locale/la/LC_MESSAGES/djangojs.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-08-02 16:29+0200\n"
+"POT-Creation-Date: 2021-01-11 21:30+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -17,14 +17,18 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#: static/js/main.js:21
+#: static/js/main.js:15
 msgid "Today"
 msgstr ""
 
-#: static/js/main.js:22
+#: static/js/main.js:16
 msgid "Cancel"
 msgstr ""
 
-#: static/js/main.js:23
+#: static/js/main.js:17
 msgid "OK"
 msgstr ""
+
+#: static/js/main.js:118
+msgid "This page may contain outdated information since there is no internet connection."
+msgstr ""

aleksis/core/locale/nb_NO/LC_MESSAGES/djangojs.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-08-02 16:29+0200\n"
+"POT-Creation-Date: 2021-01-11 21:30+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -17,14 +17,18 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#: static/js/main.js:21
+#: static/js/main.js:15
 msgid "Today"
 msgstr ""
 
-#: static/js/main.js:22
+#: static/js/main.js:16
 msgid "Cancel"
 msgstr ""
 
-#: static/js/main.js:23
+#: static/js/main.js:17
 msgid "OK"
 msgstr ""
+
+#: static/js/main.js:118
+msgid "This page may contain outdated information since there is no internet connection."
+msgstr ""