diff --git a/aleksis/core/models.py b/aleksis/core/models.py
index d969851a279174635d331f2a567c09715e35f00b..271161c8b4e47d902efbc4b3d8bd0e77d63c4cae 100644
--- a/aleksis/core/models.py
+++ b/aleksis/core/models.py
@@ -336,20 +336,29 @@ class Person(ExtensibleModel):
 
     def save(self, *args, **kwargs):
         # Determine all fields that were changed since last load
-        dirty = self.pk is None or bool(self.user_info_tracker.changed())
+        changed = self.user_info_tracker.changed()
 
         super().save(*args, **kwargs)
 
-        if self.user and dirty:
-            # Synchronise user fields to linked User object to keep it up to date
-            self.user.first_name = self.first_name
-            self.user.last_name = self.last_name
-            self.user.email = self.email
-            self.user.save()
-
-            # Synchronise groups to Django
-            for group in self.member_of.all():
-                group.save()
+        if self.pk is None or bool(changed):
+            if "user" in changed:
+                # Clear groups of previous Django user
+                previous_user = changed["user"]
+                if previous_user is not None:
+                    get_user_model().objects.get(pk=previous_user).groups.clear()
+
+            if self.user:
+                if "first_name" in changed or "last_name" in changed or "email" in changed:
+                    # Synchronise user fields to linked User object to keep it up to date
+                    self.user.first_name = self.first_name
+                    self.user.last_name = self.last_name
+                    self.user.email = self.email
+                    self.user.save()
+
+                if "user" in changed:
+                    # Synchronise groups to Django groups
+                    for group in self.member_of.all():
+                        group.save(force=True)
 
         # Select a primary group if none is set
         self.auto_select_primary_group()