From 0fb8bd9edb022b6a311b5ea93f92d4e43c399559 Mon Sep 17 00:00:00 2001
From: Jonathan Weth <git@jonathanweth.de>
Date: Thu, 16 Apr 2020 14:58:16 +0200
Subject: [PATCH] Use tqdm for progress bar
---
.../untis/util/mysql/importers/absences.py | 7 +++---
.../untis/util/mysql/importers/common_data.py | 24 ++++++++++---------
.../apps/untis/util/mysql/importers/events.py | 6 +++--
.../untis/util/mysql/importers/holidays.py | 5 ++--
.../untis/util/mysql/importers/lessons.py | 5 ++--
.../util/mysql/importers/substitutions.py | 5 ++--
aleksis/apps/untis/util/mysql/util.py | 5 ++++
pyproject.toml | 1 +
8 files changed, 36 insertions(+), 22 deletions(-)
diff --git a/aleksis/apps/untis/util/mysql/importers/absences.py b/aleksis/apps/untis/util/mysql/importers/absences.py
index 05de71f..d6b7d45 100644
--- a/aleksis/apps/untis/util/mysql/importers/absences.py
+++ b/aleksis/apps/untis/util/mysql/importers/absences.py
@@ -1,12 +1,14 @@
import logging
+from tqdm import tqdm
+
from aleksis.apps.chronos import models as chronos_models
from .... import models as mysql_models
from ..util import (
run_default_filter,
get_term,
- untis_date_to_date, move_weekday_to_range, get_first_period, get_last_period,
+ untis_date_to_date, move_weekday_to_range, get_first_period, get_last_period, TQDM_DEFAULTS,
)
logger = logging.getLogger(__name__)
@@ -31,13 +33,12 @@ def import_absences(
)
existing_absences = []
- for absence in absences:
+ for absence in tqdm(absences, desc="Import absences", **TQDM_DEFAULTS):
import_ref = absence.absence_id
logger.info("Import absence {}".format(import_ref))
if absence.absence_reason_id == 0:
- logger.warning(" Absence reason needed")
reason = unknown_reason
else:
reason = absence_reasons_ref[absence.absence_reason_id]
diff --git a/aleksis/apps/untis/util/mysql/importers/common_data.py b/aleksis/apps/untis/util/mysql/importers/common_data.py
index d867f2c..181ce76 100644
--- a/aleksis/apps/untis/util/mysql/importers/common_data.py
+++ b/aleksis/apps/untis/util/mysql/importers/common_data.py
@@ -3,12 +3,14 @@ from datetime import time
from typing import List, Dict
from constance import config
+from tqdm import tqdm
from aleksis.apps.chronos import models as chronos_models
from aleksis.core import models as core_models
from .... import models as mysql_models
-from ..util import run_default_filter, untis_colour_to_hex, untis_split_first, sync_m2m, connect_untis_fields
+from ..util import run_default_filter, untis_colour_to_hex, untis_split_first, sync_m2m, connect_untis_fields, \
+ TQDM_DEFAULTS
logger = logging.getLogger(__name__)
@@ -21,7 +23,7 @@ def import_subjects() -> Dict[int, chronos_models.Subject]:
# Get subjects
subjects = run_default_filter(mysql_models.Subjects.objects, filter_term=False)
- for subject in subjects:
+ for subject in tqdm(subjects, desc="Import subjects", **TQDM_DEFAULTS):
# Check if needed data are provided
if not subject.name:
logger.error(
@@ -90,7 +92,7 @@ def import_teachers() -> Dict[int, core_models.Person]:
# Get teachers
teachers = run_default_filter(mysql_models.Teacher.objects)
- for teacher in teachers:
+ for teacher in tqdm(teachers, desc="Import teachers", **TQDM_DEFAULTS):
# Check if needed data are provided
if not teacher.name:
logger.error(
@@ -160,7 +162,7 @@ def import_classes(
# Get classes
course_classes = run_default_filter(mysql_models.Class.objects, filter_term=True)
- for class_ in course_classes:
+ for class_ in tqdm(course_classes, desc="Import classes", **TQDM_DEFAULTS):
# Check if needed data are provided
if not class_.name:
logger.error(
@@ -229,7 +231,7 @@ def import_rooms() -> Dict[int, chronos_models.Room]:
# Get rooms
rooms = run_default_filter(mysql_models.Room.objects)
- for room in rooms:
+ for room in tqdm(rooms, desc="Import rooms", **TQDM_DEFAULTS):
if not room.name:
logger.error(
"Room ID {}: Cannot import room without short name.".format(
@@ -283,7 +285,7 @@ def import_supervision_areas(
# Get supervision areas
areas = run_default_filter(mysql_models.Corridor.objects, filter_term=False)
- for area in areas:
+ for area in tqdm(areas, desc="Import supervision areas", **TQDM_DEFAULTS):
if not area.name:
logger.error(
"Supervision area ID {}: Cannot import supervision area without short name.".format(
@@ -403,7 +405,7 @@ def import_time_periods() -> Dict[int, Dict[int, chronos_models.TimePeriod]]:
times = run_default_filter(mysql_models.Commondata.objects, filter_term=False).filter(id=30).order_by("number")
times_ref = {}
- for time_ in times:
+ for time_ in tqdm(times, desc="Import times", **TQDM_DEFAULTS):
period = time_.number
# Extract time
@@ -419,7 +421,7 @@ def import_time_periods() -> Dict[int, Dict[int, chronos_models.TimePeriod]]:
)
time_periods_ref = {}
- for time_period in periods:
+ for time_period in tqdm(periods, desc="Import time periods", **TQDM_DEFAULTS):
weekday = time_period.number - 1
period = time_period.number1
start_time = times_ref[period][0]
@@ -460,12 +462,12 @@ def import_breaks(
) -> Dict[int, Dict[int, chronos_models.Break]]:
# Build breaks for all weekdays
breaks_ref = {}
- for weekday, time_periods in time_periods_ref.items():
+ for weekday, time_periods in tqdm(time_periods_ref.items(), desc="Import breaks (weekday)", **TQDM_DEFAULTS):
breaks_ref[weekday] = {}
# Add None two times in order to create breaks before first lesson and after last lesson
time_periods_for_breaks = [None] + list(time_periods.values()) + [None]
- for i, time_period in enumerate(time_periods_for_breaks):
+ for i, time_period in tqdm(enumerate(time_periods_for_breaks), desc="Import breaks (period)", **TQDM_DEFAULTS):
# If last item (None) is reached, no further break must be created
if i + 1 == len(time_periods_for_breaks):
break
@@ -505,7 +507,7 @@ def import_absence_reasons() -> Dict[int, chronos_models.AbsenceReason]:
# Get reasons
reasons = run_default_filter(mysql_models.Absencereason.objects, filter_term=False)
- for reason in reasons:
+ for reason in tqdm(reasons, desc="Import absence reasons", **TQDM_DEFAULTS):
if not reason.name:
logger.error(
"Absence reason ID {}: Cannot import absence reason without short name.".format(
diff --git a/aleksis/apps/untis/util/mysql/importers/events.py b/aleksis/apps/untis/util/mysql/importers/events.py
index 9c0dccc..51b285b 100644
--- a/aleksis/apps/untis/util/mysql/importers/events.py
+++ b/aleksis/apps/untis/util/mysql/importers/events.py
@@ -1,5 +1,7 @@
import logging
+from tqdm import tqdm
+
from aleksis.apps.chronos import models as chronos_models
from .... import models as mysql_models
@@ -10,7 +12,7 @@ from ..util import (
move_weekday_to_range,
get_first_period,
get_last_period,
- connect_untis_fields, sync_m2m,
+ connect_untis_fields, sync_m2m, TQDM_DEFAULTS,
)
logger = logging.getLogger(__name__)
@@ -32,7 +34,7 @@ def import_events(time_periods_ref, teachers_ref, classes_ref, rooms_ref):
)
existing_events = []
- for event in events:
+ for event in tqdm(events, desc="Import events", **TQDM_DEFAULTS):
import_ref = event.event_id
logger.info("Import event {}".format(import_ref))
diff --git a/aleksis/apps/untis/util/mysql/importers/holidays.py b/aleksis/apps/untis/util/mysql/importers/holidays.py
index fe85b74..004acc2 100644
--- a/aleksis/apps/untis/util/mysql/importers/holidays.py
+++ b/aleksis/apps/untis/util/mysql/importers/holidays.py
@@ -1,11 +1,12 @@
import logging
from typing import Dict
+from tqdm import tqdm
from aleksis.apps.chronos import models as chronos_models
from .... import models as mysql_models
-from ..util import run_default_filter, untis_date_to_date
+from ..util import run_default_filter, untis_date_to_date, TQDM_DEFAULTS
logger = logging.getLogger(__name__)
@@ -17,7 +18,7 @@ def import_holidays() -> Dict[int, chronos_models.Holiday]:
# Get holidays
holidays = run_default_filter(mysql_models.Holiday.objects, filter_term=False)
- for holiday in holidays:
+ for holiday in tqdm(holidays, desc="Import holidays", **TQDM_DEFAULTS):
import_ref = holiday.holiday_id
# Check if needed data are provided
diff --git a/aleksis/apps/untis/util/mysql/importers/lessons.py b/aleksis/apps/untis/util/mysql/importers/lessons.py
index 5418c01..9e70b97 100644
--- a/aleksis/apps/untis/util/mysql/importers/lessons.py
+++ b/aleksis/apps/untis/util/mysql/importers/lessons.py
@@ -3,6 +3,7 @@ from datetime import timedelta
from constance import config
from django.utils.translation import gettext as _
+from tqdm import tqdm
from aleksis.apps.chronos import models as chronos_models
from aleksis.core import models as core_models
@@ -16,7 +17,7 @@ from ..util import (
get_term,
sync_m2m,
compare_m2m,
- connect_untis_fields,
+ connect_untis_fields, TQDM_DEFAULTS,
)
logger = logging.getLogger(__name__)
@@ -44,7 +45,7 @@ def import_lessons(
# Lessons
lessons = run_default_filter(mysql_models.Lesson.objects)
- for lesson in lessons:
+ for lesson in tqdm(lessons, desc="Import lessons", **TQDM_DEFAULTS):
lesson_id = lesson.lesson_id
messages.info(None, message=_("Import lesson {}").format(lesson_id))
diff --git a/aleksis/apps/untis/util/mysql/importers/substitutions.py b/aleksis/apps/untis/util/mysql/importers/substitutions.py
index 351100d..32a9bb9 100644
--- a/aleksis/apps/untis/util/mysql/importers/substitutions.py
+++ b/aleksis/apps/untis/util/mysql/importers/substitutions.py
@@ -2,6 +2,7 @@ import logging
from calendarweek import CalendarWeek
from django.db.models import Q
+from tqdm import tqdm
from aleksis.apps.chronos import models as chronos_models
from ..util import (
@@ -9,7 +10,7 @@ from ..util import (
untis_split_first,
untis_date_to_date,
sync_m2m,
- get_term,
+ get_term, TQDM_DEFAULTS,
)
from .... import models as mysql_models
@@ -38,7 +39,7 @@ def import_substitutions(
)
existing_subs = []
- for sub in subs:
+ for sub in tqdm(subs, desc="Import substitutions", **TQDM_DEFAULTS):
# IDs
sub_id = sub.substitution_id
existing_subs.append(sub_id)
diff --git a/aleksis/apps/untis/util/mysql/util.py b/aleksis/apps/untis/util/mysql/util.py
index 128efa8..ac16e9f 100644
--- a/aleksis/apps/untis/util/mysql/util.py
+++ b/aleksis/apps/untis/util/mysql/util.py
@@ -10,6 +10,11 @@ DB_NAME = "untis"
logger = logging.getLogger(__name__)
+TQDM_DEFAULTS = {
+ "disable": None,
+ "unit": "obj",
+ "dynamic_ncols": True,
+}
def run_using(obj: QuerySet) -> QuerySet:
return obj.using(DB_NAME)
diff --git a/pyproject.toml b/pyproject.toml
index 8fc6c3a..d6beadf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -23,6 +23,7 @@ python = "^3.7"
AlekSIS = { path = "../../.." }
AlekSIS-App-Chronos = { path = "../AlekSIS-App-Chronos" }
mysqlclient = "^1.4.6"
+tqdm = "^4.44.1"
[build-system]
requires = ["poetry>=0.12"]
--
GitLab