refactored fix duplicates

apps/location/models.py

@@ -14,7 +14,8 @@ from django.contrib.postgres.fields import ArrayField
 from translation.models import Language
 from utils.models import (ProjectBaseMixin, SVGImageMixin, TJSONField,
                           TranslatedFieldsMixin, get_current_locale,
-                          IntermediateGalleryModelMixin, GalleryModelMixin)
+                          IntermediateGalleryModelMixin, GalleryModelMixin,
+                          RelatedInstanceMixin)
 
 
 class CountryQuerySet(models.QuerySet):
@@ -24,7 +25,8 @@ class CountryQuerySet(models.QuerySet):
         return self.filter(is_active=switcher)
 
 
-class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin):
+class Country(RelatedInstanceMixin, TranslatedFieldsMixin,
+              SVGImageMixin, ProjectBaseMixin):
     """Country model."""
 
     STR_FIELD_NAME = 'name'
@@ -49,16 +51,6 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin):
 
     objects = CountryQuerySet.as_manager()
 
-    @property
-    def time_format(self):
-        if self.code.lower() not in self.TWELVE_HOURS_FORMAT_COUNTRIES:
-            return 'HH:mm'
-        return 'hh:mmA'
-
-    @property
-    def country_id(self):
-        return self.id
-
     class Meta:
         """Meta class."""
 
@@ -73,8 +65,18 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin):
                 str_name = translated_name
         return str_name
 
+    @property
+    def time_format(self):
+        if self.code.lower() not in self.TWELVE_HOURS_FORMAT_COUNTRIES:
+            return 'HH:mm'
+        return 'hh:mmA'
 
-class Region(models.Model):
+    @property
+    def country_id(self):
+        return self.id
+
+
+class Region(RelatedInstanceMixin, models.Model):
     """Region model."""
 
     name = models.CharField(_('name'), max_length=250)
@@ -117,7 +119,7 @@ class CityQuerySet(models.QuerySet):
         return self.filter(country__code=code)
 
 
-class City(GalleryModelMixin):
+class City(RelatedInstanceMixin, GalleryModelMixin):
     """Region model."""
     name = models.CharField(_('name'), max_length=250)
     name_translated = TJSONField(blank=True, null=True, default=None,
@@ -154,29 +156,6 @@ class City(GalleryModelMixin):
     def __str__(self):
         return self.name
 
-    @property
-    def _related_objects(self) -> list:
-        """Return list of related objects."""
-        related_objects = []
-        for related_object in self._meta.related_objects:
-            related_objects.append(related_object)
-        return related_objects
-
-    @property
-    def _related_instances(self) -> set:
-        """Return list of related instances."""
-        related_instances = []
-        related_names = [related_object.related_name
-                         if related_object.related_name
-                         else f'{related_object.name}_set'
-                         for related_object in self._related_objects]
-        for related_object in related_names:
-            instances = getattr(self, f'{related_object}')
-            if instances.exists():
-                for instance in instances.all():
-                    related_instances.append(instance)
-        return set(related_instances)
-
 
 class CityGallery(IntermediateGalleryModelMixin):
     """Gallery for model City."""

apps/location/transfer_data.py

@@ -17,7 +17,7 @@ from review.models import Review
 from tag.models import TagCategory, ChosenTagSettings
 from transfer import models as transfer_models
 from transfer.serializers import location as location_serializers
-from transfer.utils import clean_old_records
+from transfer.utils import clean_old_records, clean_old_country_records, clean_old_region_records
 
 
 def transfer_countries():
@@ -497,6 +497,7 @@ def fix_location_models():
         ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "r")
         ruby_data = json.loads(ruby_data_file.read())
     except FileNotFoundError:
+        print('create ruby data')
         ruby_data = get_ruby_data()
 
     print('add_correct_location_models')
@@ -521,7 +522,8 @@ def fix_location_models():
 
 def remove_old_records():
     clean_old_records(City, {"mysql_id__isnull": True})
-    clean_old_records(Region, {"mysql_ids__isnull": True})
+    clean_old_country_records(Country, {"mysql_ids__isnull": True})
+    clean_old_region_records(Region, {"mysql_ids__isnull": True})
 
 
 def transfer_city_gallery():
@@ -752,7 +754,8 @@ data_types = {
         migrate_city_photos
     ],
     "fix_location": [
-        fix_location_models
+        add_fake_country,
+        fix_location_models,
     ],
     "remove_old_locations": [
         remove_old_records
@@ -761,7 +764,7 @@ data_types = {
         transfer_city_gallery
     ],
     "add_fake_country": [
-        add_fake_country
+        add_fake_country,
     ],
 
     "setup_clean_db": [setup_clean_db],

apps/transfer/utils.py

@@ -2,6 +2,8 @@ from os.path import exists
 from importlib.machinery import SourceFileLoader
 from django.apps import apps
 from django.conf import settings
+from django.db.models import Count, Q
+from tqdm import tqdm
 
 
 def transfer_objects(data_type):
@@ -24,6 +26,7 @@ def transfer_objects(data_type):
 def clean_old_records(model, conditions):
     error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w")
     non_existed_correct_city_ids = []
+    to_delete = []
 
     try:
         old_records = model.objects.filter(**conditions)
@@ -31,12 +34,14 @@ def clean_old_records(model, conditions):
         error_file.write(f"Cannot find model objects: {e}")
         return
 
-    for old_record in old_records:
+    for old_record in tqdm(old_records):
         correct_record_qs = model.objects.exclude(id=old_record.id) \
                                          .exclude(**conditions) \
-                                         .filter(name=old_record.name)
+                                         .filter(name=old_record.name,
+                                                 mysql_id=old_record.old_id)
         if correct_record_qs.exists():
             correct_record = correct_record_qs.first()
+            # check object dependencies
             for rel_instance in old_record._related_instances:
                 field_name = [related_field.name
                               for related_field in rel_instance._meta.fields
@@ -45,14 +50,119 @@ def clean_old_records(model, conditions):
                 if getattr(rel_instance, field_name) != correct_record:
                     setattr(rel_instance, field_name, correct_record)
                     rel_instance.save()
+            to_delete.append(old_record.id)
         else:
             non_existed_correct_city_ids.append(old_record.id)
 
     if non_existed_correct_city_ids:
         print(f"Non existed correct city ids: {non_existed_correct_city_ids}")
-        return
 
     # delete old records
-    counter = old_records.count()
+    counter = len(to_delete)
-    old_records.delete()
+    old_records.filter(id__in=to_delete).delete()
+    print(f'Deleted {counter} objects.')
+
+
+def clean_old_country_records(model, conditions):
+    error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w")
+    non_existed_correct_country_ids = []
+    to_delete = []
+
+    try:
+        unique_codes = model.objects.values_list('code', flat=True) \
+                                    .annotate(counter=Count('code')) \
+                                    .filter(counter=1) \
+                                    .values_list('code', flat=True)
+        old_records = model.objects.exclude(code__in=unique_codes) \
+                                   .filter(**conditions)
+    except Exception as e:
+        error_file.write(f"Cannot find model objects: {e}")
+        return
+
+    for old_record in tqdm(old_records):
+        correct_record_qs = model.objects.exclude(id=old_record.id) \
+                                         .exclude(**conditions) \
+                                         .filter(Q(code=old_record.code) |
+                                                 Q(mysql_ids__contains=[old_record.old_id, ]))
+        if correct_record_qs.exists():
+            correct_record = correct_record_qs.first()
+            # check object dependencies
+            for rel_instance in old_record._related_instances:
+                if not hasattr(rel_instance, '_meta'):
+                    for related in rel_instance.all():
+                        field_name = [related_field.name
+                                      for related_field in related._meta.fields
+                                      if related_field.related_model == correct_record._meta.model][0]
+                        # rebinding correct dependency instances
+                        if getattr(rel_instance, field_name) != correct_record:
+                            setattr(rel_instance, field_name, correct_record)
+                            rel_instance.save()
+                else:
+                    field_name = [related_field.name
+                                  for related_field in rel_instance._meta.fields
+                                  if related_field.related_model == correct_record._meta.model][0]
+                    # rebinding correct dependency instances
+                    if getattr(rel_instance, field_name) != correct_record:
+                        setattr(rel_instance, field_name, correct_record)
+                        rel_instance.save()
+            to_delete.append(old_record.id)
+        else:
+            non_existed_correct_country_ids.append(old_record.id)
+
+    if non_existed_correct_country_ids:
+        print(f"Non existed correct country ids: {non_existed_correct_country_ids}")
+
+    # delete old records
+    counter = len(to_delete)
+    old_records.filter(id__in=to_delete).delete()
+    print(f'Deleted {counter} objects.')
+
+
+def clean_old_region_records(model, conditions):
+    error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w")
+    non_existed_correct_country_ids = []
+    to_delete = []
+
+    try:
+        old_records = model.objects.filter(**conditions)
+    except Exception as e:
+        error_file.write(f"Cannot find model objects: {e}")
+        return
+
+    for old_record in tqdm(old_records):
+        correct_record_qs = model.objects.exclude(id=old_record.id) \
+                                         .exclude(**conditions) \
+                                         .filter(code__iexact=old_record.code,
+                                                 mysql_ids__contains=[old_record.old_id, ])
+        if correct_record_qs.exists():
+            correct_record = correct_record_qs.first()
+            # check object dependencies
+            for rel_instance in old_record._related_instances:
+                if not hasattr(rel_instance, '_meta'):
+                    for related in rel_instance.all():
+                        field_name = [related_field.name
+                                      for related_field in related._meta.fields
+                                      if related_field.related_model == correct_record._meta.model][0]
+                        # rebinding correct dependency instances
+                        if getattr(rel_instance, field_name) != correct_record:
+                            setattr(rel_instance, field_name, correct_record)
+                            rel_instance.save()
+                else:
+                    field_name = [related_field.name
+                                  for related_field in rel_instance._meta.fields
+                                  if related_field.related_model == correct_record._meta.model][0]
+                    # rebinding correct dependency instances
+                    if getattr(rel_instance, field_name) != correct_record:
+                        setattr(rel_instance, field_name, correct_record)
+                        rel_instance.save()
+            to_delete.append(old_record.id)
+        else:
+            non_existed_correct_country_ids.append(old_record.id)
+
+    if non_existed_correct_country_ids:
+        print(f"Non existed correct region ids: {non_existed_correct_country_ids}")
+
+    # delete old records
+    counter = len(to_delete)
+    old_records.filter(id__in=to_delete).delete()
     print(f'Deleted {counter} objects.')

apps/utils/models.py

@@ -452,4 +452,40 @@ class FavoritesMixin:
         return self.favorites.aggregate(arr=ArrayAgg('user_id')).get('arr')
 
 
+class RelatedInstanceMixin:
+    """Mixin for getting related objects."""
+
+    @property
+    def _related_objects(self) -> list:
+        """Return list of related objects."""
+        if hasattr(self, '_meta'):
+            related_objects = []
+            for related_object in self._meta.related_objects:
+                related_objects.append(related_object)
+            return related_objects
+
+    @property
+    def _related_instances(self) -> set:
+        """Return list of related instances."""
+        if hasattr(self, '_related_objects'):
+            related_instances = []
+            related_names = [related_object.related_name or f'{related_object.name}_set'
+                             if related_object.multiple
+                             else f'{related_object.name}'
+                             for related_object in self._related_objects]
+            for related_object in related_names:
+                try:
+                    instances = getattr(self, f'{related_object}')
+                except:
+                    continue
+                if hasattr(instances, 'exists') and instances.exists():
+                    for instance in instances.all():
+                        related_instances.append(instance)
+                else:
+                    # if one object put it in list.
+                    related_instances.append(instances)
+
+            return set(related_instances)
+
+
 timezone.datetime.now().date().isoformat()