diff --git a/apps/location/models.py b/apps/location/models.py index da202224..10d578b8 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -14,7 +14,8 @@ from django.contrib.postgres.fields import ArrayField from translation.models import Language from utils.models import (ProjectBaseMixin, SVGImageMixin, TJSONField, TranslatedFieldsMixin, get_current_locale, - IntermediateGalleryModelMixin, GalleryModelMixin) + IntermediateGalleryModelMixin, GalleryModelMixin, + RelatedInstanceMixin) class CountryQuerySet(models.QuerySet): @@ -24,7 +25,8 @@ class CountryQuerySet(models.QuerySet): return self.filter(is_active=switcher) -class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin): +class Country(RelatedInstanceMixin, TranslatedFieldsMixin, + SVGImageMixin, ProjectBaseMixin): """Country model.""" STR_FIELD_NAME = 'name' @@ -49,16 +51,6 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin): objects = CountryQuerySet.as_manager() - @property - def time_format(self): - if self.code.lower() not in self.TWELVE_HOURS_FORMAT_COUNTRIES: - return 'HH:mm' - return 'hh:mmA' - - @property - def country_id(self): - return self.id - class Meta: """Meta class.""" @@ -73,8 +65,18 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin): str_name = translated_name return str_name + @property + def time_format(self): + if self.code.lower() not in self.TWELVE_HOURS_FORMAT_COUNTRIES: + return 'HH:mm' + return 'hh:mmA' -class Region(models.Model): + @property + def country_id(self): + return self.id + + +class Region(RelatedInstanceMixin, models.Model): """Region model.""" name = models.CharField(_('name'), max_length=250) @@ -117,7 +119,7 @@ class CityQuerySet(models.QuerySet): return self.filter(country__code=code) -class City(GalleryModelMixin): +class City(RelatedInstanceMixin, GalleryModelMixin): """Region model.""" name = models.CharField(_('name'), max_length=250) name_translated = TJSONField(blank=True, null=True, default=None, @@ -154,29 +156,6 @@ class City(GalleryModelMixin): def __str__(self): return self.name - @property - def _related_objects(self) -> list: - """Return list of related objects.""" - related_objects = [] - for related_object in self._meta.related_objects: - related_objects.append(related_object) - return related_objects - - @property - def _related_instances(self) -> set: - """Return list of related instances.""" - related_instances = [] - related_names = [related_object.related_name - if related_object.related_name - else f'{related_object.name}_set' - for related_object in self._related_objects] - for related_object in related_names: - instances = getattr(self, f'{related_object}') - if instances.exists(): - for instance in instances.all(): - related_instances.append(instance) - return set(related_instances) - class CityGallery(IntermediateGalleryModelMixin): """Gallery for model City.""" diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index c027f4a1..c41eae74 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -17,7 +17,7 @@ from review.models import Review from tag.models import TagCategory, ChosenTagSettings from transfer import models as transfer_models from transfer.serializers import location as location_serializers -from transfer.utils import clean_old_records +from transfer.utils import clean_old_records, clean_old_country_records, clean_old_region_records def transfer_countries(): @@ -497,6 +497,7 @@ def fix_location_models(): ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "r") ruby_data = json.loads(ruby_data_file.read()) except FileNotFoundError: + print('create ruby data') ruby_data = get_ruby_data() print('add_correct_location_models') @@ -521,7 +522,8 @@ def fix_location_models(): def remove_old_records(): clean_old_records(City, {"mysql_id__isnull": True}) - clean_old_records(Region, {"mysql_ids__isnull": True}) + clean_old_country_records(Country, {"mysql_ids__isnull": True}) + clean_old_region_records(Region, {"mysql_ids__isnull": True}) def transfer_city_gallery(): @@ -752,7 +754,8 @@ data_types = { migrate_city_photos ], "fix_location": [ - fix_location_models + add_fake_country, + fix_location_models, ], "remove_old_locations": [ remove_old_records @@ -761,7 +764,7 @@ data_types = { transfer_city_gallery ], "add_fake_country": [ - add_fake_country + add_fake_country, ], "setup_clean_db": [setup_clean_db], diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index d3f3b5bd..20e61f18 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -2,6 +2,8 @@ from os.path import exists from importlib.machinery import SourceFileLoader from django.apps import apps from django.conf import settings +from django.db.models import Count, Q +from tqdm import tqdm def transfer_objects(data_type): @@ -24,6 +26,7 @@ def transfer_objects(data_type): def clean_old_records(model, conditions): error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w") non_existed_correct_city_ids = [] + to_delete = [] try: old_records = model.objects.filter(**conditions) @@ -31,12 +34,14 @@ def clean_old_records(model, conditions): error_file.write(f"Cannot find model objects: {e}") return - for old_record in old_records: + for old_record in tqdm(old_records): correct_record_qs = model.objects.exclude(id=old_record.id) \ .exclude(**conditions) \ - .filter(name=old_record.name) + .filter(name=old_record.name, + mysql_id=old_record.old_id) if correct_record_qs.exists(): correct_record = correct_record_qs.first() + # check object dependencies for rel_instance in old_record._related_instances: field_name = [related_field.name for related_field in rel_instance._meta.fields @@ -45,14 +50,119 @@ def clean_old_records(model, conditions): if getattr(rel_instance, field_name) != correct_record: setattr(rel_instance, field_name, correct_record) rel_instance.save() + to_delete.append(old_record.id) else: non_existed_correct_city_ids.append(old_record.id) if non_existed_correct_city_ids: print(f"Non existed correct city ids: {non_existed_correct_city_ids}") - return # delete old records - counter = old_records.count() - old_records.delete() + counter = len(to_delete) + old_records.filter(id__in=to_delete).delete() + print(f'Deleted {counter} objects.') + + +def clean_old_country_records(model, conditions): + error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w") + non_existed_correct_country_ids = [] + to_delete = [] + + try: + unique_codes = model.objects.values_list('code', flat=True) \ + .annotate(counter=Count('code')) \ + .filter(counter=1) \ + .values_list('code', flat=True) + old_records = model.objects.exclude(code__in=unique_codes) \ + .filter(**conditions) + except Exception as e: + error_file.write(f"Cannot find model objects: {e}") + return + + for old_record in tqdm(old_records): + correct_record_qs = model.objects.exclude(id=old_record.id) \ + .exclude(**conditions) \ + .filter(Q(code=old_record.code) | + Q(mysql_ids__contains=[old_record.old_id, ])) + if correct_record_qs.exists(): + correct_record = correct_record_qs.first() + # check object dependencies + for rel_instance in old_record._related_instances: + if not hasattr(rel_instance, '_meta'): + for related in rel_instance.all(): + field_name = [related_field.name + for related_field in related._meta.fields + if related_field.related_model == correct_record._meta.model][0] + # rebinding correct dependency instances + if getattr(rel_instance, field_name) != correct_record: + setattr(rel_instance, field_name, correct_record) + rel_instance.save() + else: + field_name = [related_field.name + for related_field in rel_instance._meta.fields + if related_field.related_model == correct_record._meta.model][0] + # rebinding correct dependency instances + if getattr(rel_instance, field_name) != correct_record: + setattr(rel_instance, field_name, correct_record) + rel_instance.save() + to_delete.append(old_record.id) + else: + non_existed_correct_country_ids.append(old_record.id) + + if non_existed_correct_country_ids: + print(f"Non existed correct country ids: {non_existed_correct_country_ids}") + + # delete old records + counter = len(to_delete) + old_records.filter(id__in=to_delete).delete() + print(f'Deleted {counter} objects.') + + +def clean_old_region_records(model, conditions): + error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w") + non_existed_correct_country_ids = [] + to_delete = [] + + try: + old_records = model.objects.filter(**conditions) + except Exception as e: + error_file.write(f"Cannot find model objects: {e}") + return + + for old_record in tqdm(old_records): + correct_record_qs = model.objects.exclude(id=old_record.id) \ + .exclude(**conditions) \ + .filter(code__iexact=old_record.code, + mysql_ids__contains=[old_record.old_id, ]) + if correct_record_qs.exists(): + correct_record = correct_record_qs.first() + # check object dependencies + for rel_instance in old_record._related_instances: + if not hasattr(rel_instance, '_meta'): + for related in rel_instance.all(): + field_name = [related_field.name + for related_field in related._meta.fields + if related_field.related_model == correct_record._meta.model][0] + # rebinding correct dependency instances + if getattr(rel_instance, field_name) != correct_record: + setattr(rel_instance, field_name, correct_record) + rel_instance.save() + else: + field_name = [related_field.name + for related_field in rel_instance._meta.fields + if related_field.related_model == correct_record._meta.model][0] + # rebinding correct dependency instances + if getattr(rel_instance, field_name) != correct_record: + setattr(rel_instance, field_name, correct_record) + rel_instance.save() + to_delete.append(old_record.id) + else: + non_existed_correct_country_ids.append(old_record.id) + + if non_existed_correct_country_ids: + print(f"Non existed correct region ids: {non_existed_correct_country_ids}") + + # delete old records + counter = len(to_delete) + old_records.filter(id__in=to_delete).delete() print(f'Deleted {counter} objects.') diff --git a/apps/utils/models.py b/apps/utils/models.py index b4b64d9f..d029338c 100644 --- a/apps/utils/models.py +++ b/apps/utils/models.py @@ -452,4 +452,40 @@ class FavoritesMixin: return self.favorites.aggregate(arr=ArrayAgg('user_id')).get('arr') +class RelatedInstanceMixin: + """Mixin for getting related objects.""" + + @property + def _related_objects(self) -> list: + """Return list of related objects.""" + if hasattr(self, '_meta'): + related_objects = [] + for related_object in self._meta.related_objects: + related_objects.append(related_object) + return related_objects + + @property + def _related_instances(self) -> set: + """Return list of related instances.""" + if hasattr(self, '_related_objects'): + related_instances = [] + related_names = [related_object.related_name or f'{related_object.name}_set' + if related_object.multiple + else f'{related_object.name}' + for related_object in self._related_objects] + for related_object in related_names: + try: + instances = getattr(self, f'{related_object}') + except: + continue + if hasattr(instances, 'exists') and instances.exists(): + for instance in instances.all(): + related_instances.append(instance) + else: + # if one object put it in list. + related_instances.append(instances) + + return set(related_instances) + + timezone.datetime.now().date().isoformat() \ No newline at end of file