From 403848a1757ee2df34c6bb16d1abe5559f61cfa2 Mon Sep 17 00:00:00 2001
From: Jonathan Weth <git@jonathanweth.de>
Date: Tue, 9 Jun 2020 20:17:44 +0200
Subject: [PATCH] Add universally usable progress bar for Celery tasks
Close #287
---
aleksis/core/settings.py | 2 +-
aleksis/core/static/js/progress.js | 67 +++++++++++++
aleksis/core/templates/core/progress.html | 46 +++++++++
aleksis/core/urls.py | 4 +
aleksis/core/util/celery_progress.py | 34 +++++++
aleksis/core/util/core_helpers.py | 116 +++++++++++++++++++++-
6 files changed, 266 insertions(+), 3 deletions(-)
create mode 100644 aleksis/core/static/js/progress.js
create mode 100644 aleksis/core/templates/core/progress.html
create mode 100644 aleksis/core/util/celery_progress.py
diff --git a/aleksis/core/settings.py b/aleksis/core/settings.py
index 756b14f7b..d8b50c495 100644
--- a/aleksis/core/settings.py
+++ b/aleksis/core/settings.py
@@ -419,7 +419,7 @@ if _settings.get("twilio.sid", None):
TWILIO_CALLER_ID = _settings.get("twilio.callerid")
if _settings.get("celery.enabled", False):
- INSTALLED_APPS += ("django_celery_beat", "django_celery_results")
+ INSTALLED_APPS += ("django_celery_beat", "django_celery_results", "celery_progress")
CELERY_BROKER_URL = _settings.get("celery.broker", "redis://localhost")
CELERY_RESULT_BACKEND = "django-db"
CELERY_CACHE_BACKEND = "django-cache"
diff --git a/aleksis/core/static/js/progress.js b/aleksis/core/static/js/progress.js
new file mode 100644
index 000000000..4c3e52c98
--- /dev/null
+++ b/aleksis/core/static/js/progress.js
@@ -0,0 +1,67 @@
+const OPTIONS = getJSONScript("progress_options");
+
+const STYLE_CLASSES = {
+ 10: 'info',
+ 20: 'info',
+ 25: 'success',
+ 30: 'warning',
+ 40: 'error',
+};
+
+const ICONS = {
+ 10: 'info',
+ 20: 'info',
+ 25: 'check_circle',
+ 30: 'warning',
+ 40: 'error',
+};
+
+function setProgress(progress) {
+ $("#progress-bar").css("width", progress + "%");
+}
+
+function renderMessageBox(level, text) {
+ return '<div class="alert ' + STYLE_CLASSES[level] + '"><p><i class="material-icons left">' + ICONS[level] + '</i>' + text + '</p></div>';
+}
+
+function customProgress(progressBarElement, progressBarMessageElement, progress) {
+ setProgress(progress.percent);
+
+ if (progress.hasOwnProperty("messages")) {
+ const messagesBox = $("#messages");
+
+ // Clear container
+ messagesBox.html("")
+
+ // Render message boxes
+ $.each(progress.messages, function (i, message) {
+ messagesBox.append(renderMessageBox(message[0], message[1]));
+ })
+ }
+}
+
+
+function customSuccess(progressBarElement, progressBarMessageElement) {
+ setProgress(100);
+ $("#result-alert").addClass("success");
+ $("#result-icon").text("check_circle");
+ $("#result-text").text(OPTIONS.success);
+ $("#result-box").show();
+}
+
+function customError(progressBarElement, progressBarMessageElement) {
+ setProgress(100);
+ $("#result-alert").addClass("error");
+ $("#result-icon").text("error");
+ $("#result-text").text(OPTIONS.error);
+ $("#result-box").show();
+}
+
+$(document).ready(function () {
+ var progressUrl = Urls["celeryProgress:taskStatus"](OPTIONS.task_id);
+ CeleryProgressBar.initProgressBar(progressUrl, {
+ onProgress: customProgress,
+ onSuccess: customSuccess,
+ onError: customError,
+ });
+});
diff --git a/aleksis/core/templates/core/progress.html b/aleksis/core/templates/core/progress.html
new file mode 100644
index 000000000..26d078d9f
--- /dev/null
+++ b/aleksis/core/templates/core/progress.html
@@ -0,0 +1,46 @@
+{% extends "core/base.html" %}
+{% load i18n static %}
+
+{% block browser_title %}
+ {{ title }}
+{% endblock %}
+{% block page_title %}
+ {{ title }}
+{% endblock %}
+
+{% block content %}
+
+ <div class="container">
+ <div class="row">
+ <div class="progress center">
+ <div class="determinate" style="width: 0;" id="progress-bar"></div>
+ </div>
+ <h6 class="center">
+ {{ progress.title }}
+ </h6>
+ </div>
+ <div class="row">
+ <div id="messages"></div>
+
+ <div id="result-box" style="display: none;">
+ <div class="alert" id="result-alert">
+ <div>
+ <i class="material-icons left" id="result-icon">check_circle</i>
+ <p id="result-text"></p>
+ </div>
+ </div>
+
+ {% url "index" as index_url %}
+ <a class="btn waves-effect waves-light" href="{{ back_url|default:index_url }}">
+ <i class="material-icons left">arrow_back</i>
+ {% trans "Go back" %}
+ </a>
+ </div>
+ </div>
+ </div>
+
+ {{ progress|json_script:"progress_options" }}
+ <script src="{% static "js/helper.js" %}"></script>
+ <script src="{% static "celery_progress/celery_progress.js" %}"></script>
+ <script src="{% static "js/progress.js" %}"></script>
+{% endblock %}
diff --git a/aleksis/core/urls.py b/aleksis/core/urls.py
index 07736f7b8..766275b5d 100644
--- a/aleksis/core/urls.py
+++ b/aleksis/core/urls.py
@@ -12,6 +12,7 @@ from django_js_reverse.views import urls_js
from two_factor.urls import urlpatterns as tf_urls
from . import views
+from .util.core_helpers import is_celery_enabled
urlpatterns = [
path("", include("pwa.urls"), name="pwa"),
@@ -158,6 +159,9 @@ if hasattr(settings, "TWILIO_ACCOUNT_SID"):
urlpatterns += [path("", include(tf_twilio_urls))]
+if is_celery_enabled():
+ urlpatterns.append(path("celery_progress/", include("celery_progress.urls")))
+
# Serve javascript-common if in development
if settings.DEBUG:
urlpatterns.append(path("__debug__/", include(debug_toolbar.urls)))
diff --git a/aleksis/core/util/celery_progress.py b/aleksis/core/util/celery_progress.py
new file mode 100644
index 000000000..9de760eeb
--- /dev/null
+++ b/aleksis/core/util/celery_progress.py
@@ -0,0 +1,34 @@
+from decimal import Decimal
+from typing import Union
+
+from celery_progress.backend import PROGRESS_STATE, AbstractProgressRecorder
+
+
+class ProgressRecorder(AbstractProgressRecorder):
+ def __init__(self, task):
+ self.task = task
+ self.messages = []
+ self.total = 100
+ self.current = 0
+
+ def set_progress(self, current: Union[int, float], **kwargs):
+ self.current = current
+
+ percent = 0
+ if self.total > 0:
+ percent = (Decimal(current) / Decimal(self.total)) * Decimal(100)
+ percent = float(round(percent, 2))
+
+ self.task.update_state(
+ state=PROGRESS_STATE,
+ meta={
+ "current": current,
+ "total": self.total,
+ "percent": percent,
+ "messages": self.messages,
+ },
+ )
+
+ def add_message(self, level: int, message: str, **kwargs):
+ self.messages.append((level, message))
+ self.set_progress(self.current)
diff --git a/aleksis/core/util/core_helpers.py b/aleksis/core/util/core_helpers.py
index dc62a7990..635ff0226 100644
--- a/aleksis/core/util/core_helpers.py
+++ b/aleksis/core/util/core_helpers.py
@@ -1,5 +1,6 @@
import os
import pkgutil
+import time
from datetime import datetime, timedelta
from importlib import import_module
from itertools import groupby
@@ -14,6 +15,10 @@ from django.shortcuts import get_object_or_404
from django.utils import timezone
from django.utils.functional import lazy
+from django_global_request.middleware import get_request
+
+from aleksis.core.util import messages
+
def copyright_years(years: Sequence[int], seperator: str = ", ", joiner: str = "–") -> str:
"""Take a sequence of integegers and produces a string with ranges.
@@ -170,6 +175,11 @@ def has_person(obj: Union[HttpRequest, Model]) -> bool:
return True
+def is_celery_enabled():
+ """Check whether celery support is enabled."""
+ return hasattr(settings, "CELERY_RESULT_BACKEND")
+
+
def celery_optional(orig: Callable) -> Callable:
"""Add a decorator that makes Celery optional for a function.
@@ -177,13 +187,13 @@ def celery_optional(orig: Callable) -> Callable:
and calls its delay method when invoked; if not, it leaves it untouched
and it is executed synchronously.
"""
- if hasattr(settings, "CELERY_RESULT_BACKEND"):
+ if is_celery_enabled():
from ..celery import app # noqa
task = app.task(orig)
def wrapped(*args, **kwargs):
- if hasattr(settings, "CELERY_RESULT_BACKEND"):
+ if is_celery_enabled():
task.delay(*args, **kwargs)
else:
orig(*args, **kwargs)
@@ -191,6 +201,108 @@ def celery_optional(orig: Callable) -> Callable:
return wrapped
+class DummyRecorder:
+ def set_progress(self, *args, **kwargs):
+ pass
+
+ def add_message(self, level: int, message: str, **kwargs) -> Optional[Any]:
+ request = get_request()
+ return messages.add_message(request, level, message, **kwargs)
+
+
+def celery_optional_progress(orig: Callable) -> Callable:
+ """Add a decorator that makes Celery with progress bar support optional for a function.
+
+ If Celery is configured and available, it wraps the function in a Task
+ and calls its delay method when invoked; if not, it leaves it untouched
+ and it is executed synchronously.
+
+ Additionally, it adds a recorder class as first argument
+ (`ProgressRecorder` if Celery is enabled, else `DummyRecoder`).
+
+ This recorder provides the functions `set_progress` and `add_message`
+ which can be used to track the status of the task.
+ For further information, see the respective recorder classes.
+
+ How to use
+ ----------
+ 1. Write a function and include tracking methods
+
+ ::
+
+ from django.contrib import messages
+
+ from aleksis.core.util.core_helpers import celery_optional_progress
+
+ @celery_optional_progress
+ def do_something(recorder: Union[ProgressRecorder, DummyRecorder], foo, bar, baz=None):
+ # ...
+ recorder.total = len(list_with_data)
+
+ for i, item in list_with_data:
+ # ...
+ recorder.set_progress(i + 1)
+ # ...
+
+ recorder.add_message(messages.SUCCESS, "All data were imported successfully.")
+
+ 2. Track process in view:
+
+ ::
+
+ def my_view(request):
+ context = {}
+ # ...
+ result = do_something(foo, bar, baz=baz)
+
+ if result:
+ context = {
+ "title": _("Progress: Import data"),
+ "back_url": reverse("index"),
+ "progress": {
+ "task_id": result.task_id,
+ "title": _("Import objects …"),
+ "success": _("The import was done successfully."),
+ "error": _("There was a problem while importing data."),
+ },
+ }
+
+ # Render progress view
+ return render(request, "core/progress.html", context)
+
+ # Render other view if Celery isn't enabled
+ return render(request, "my-app/other-view.html", context)
+ """
+
+ def recorder_func(self, *args, **kwargs):
+ if is_celery_enabled():
+ from .celery_progress import ProgressRecorder # noqa
+
+ recorder = ProgressRecorder(self)
+ else:
+ recorder = DummyRecorder()
+ orig(recorder, *args, **kwargs)
+
+ # Needed to ensure that all messages are displayed by frontend
+ time.sleep(0.7)
+
+ var_name = f"{orig.__module__}.{orig.__name__}"
+
+ if is_celery_enabled():
+ from ..celery import app # noqa
+
+ task = app.task(recorder_func, bind=True, name=var_name)
+
+ def wrapped(*args, **kwargs):
+ if is_celery_enabled():
+ return task.delay(*args, **kwargs)
+ else:
+ recorder_func(None, *args, **kwargs)
+ return None
+
+ return wrapped
+
+
def path_and_rename(instance, filename: str, upload_to: str = "files") -> str:
"""Update path of an uploaded file and renames it to a random UUID in Django FileField."""
_, ext = os.path.splitext(filename)
--
GitLab