phzhik/gault-millau
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactored fix duplicates

Browse Source
This commit is contained in:
Anatoly 2019-12-16 13:46:40 +03:00
parent ade2d03402
commit 956aae281f
4 changed files with 174 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
apps/location/models.py
View File

@ -14,7 +14,8 @@ from django.contrib.postgres.fields import ArrayField
from translation.models import Language
from utils.models import (ProjectBaseMixin, SVGImageMixin, TJSONField,
TranslatedFieldsMixin, get_current_locale,
IntermediateGalleryModelMixin, GalleryModelMixin)
IntermediateGalleryModelMixin, GalleryModelMixin,
RelatedInstanceMixin)
class CountryQuerySet(models.QuerySet):
@ -24,7 +25,8 @@ class CountryQuerySet(models.QuerySet):
return self.filter(is_active=switcher)
class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin):
class Country(RelatedInstanceMixin, TranslatedFieldsMixin,
SVGImageMixin, ProjectBaseMixin):
"""Country model."""
STR_FIELD_NAME = 'name'
@ -49,16 +51,6 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin):
objects = CountryQuerySet.as_manager()
@property
def time_format(self):
if self.code.lower() not in self.TWELVE_HOURS_FORMAT_COUNTRIES:
return 'HH:mm'
return 'hh:mmA'
@property
def country_id(self):
return self.id
class Meta:
"""Meta class."""
@ -73,8 +65,18 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin):
str_name = translated_name
return str_name
@property
def time_format(self):
if self.code.lower() not in self.TWELVE_HOURS_FORMAT_COUNTRIES:
return 'HH:mm'
return 'hh:mmA'
class Region(models.Model):
@property
def country_id(self):
return self.id
class Region(RelatedInstanceMixin, models.Model):
"""Region model."""
name = models.CharField(_('name'), max_length=250)
@ -117,7 +119,7 @@ class CityQuerySet(models.QuerySet):
return self.filter(country__code=code)
class City(GalleryModelMixin):
class City(RelatedInstanceMixin, GalleryModelMixin):
"""Region model."""
name = models.CharField(_('name'), max_length=250)
name_translated = TJSONField(blank=True, null=True, default=None,
@ -154,29 +156,6 @@ class City(GalleryModelMixin):
def __str__(self):
return self.name
@property
def _related_objects(self) -> list:
"""Return list of related objects."""
related_objects = []
for related_object in self._meta.related_objects:
related_objects.append(related_object)
return related_objects
@property
def _related_instances(self) -> set:
"""Return list of related instances."""
related_instances = []
related_names = [related_object.related_name
if related_object.related_name
else f'{related_object.name}_set'
for related_object in self._related_objects]
for related_object in related_names:
instances = getattr(self, f'{related_object}')
if instances.exists():
for instance in instances.all():
related_instances.append(instance)
return set(related_instances)
class CityGallery(IntermediateGalleryModelMixin):
"""Gallery for model City."""

11
apps/location/transfer_data.py
View File

@ -17,7 +17,7 @@ from review.models import Review
from tag.models import TagCategory, ChosenTagSettings
from transfer import models as transfer_models
from transfer.serializers import location as location_serializers
from transfer.utils import clean_old_records
from transfer.utils import clean_old_records, clean_old_country_records, clean_old_region_records
def transfer_countries():
@ -497,6 +497,7 @@ def fix_location_models():
ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "r")
ruby_data = json.loads(ruby_data_file.read())
except FileNotFoundError:
print('create ruby data')
ruby_data = get_ruby_data()
print('add_correct_location_models')
@ -521,7 +522,8 @@ def fix_location_models():
def remove_old_records():
clean_old_records(City, {"mysql_id__isnull": True})
clean_old_records(Region, {"mysql_ids__isnull": True})
clean_old_country_records(Country, {"mysql_ids__isnull": True})
clean_old_region_records(Region, {"mysql_ids__isnull": True})
def transfer_city_gallery():
@ -752,7 +754,8 @@ data_types = {
migrate_city_photos
],
"fix_location": [
fix_location_models
add_fake_country,
fix_location_models,
],
"remove_old_locations": [
remove_old_records
@ -761,7 +764,7 @@ data_types = {
transfer_city_gallery
],
"add_fake_country": [
add_fake_country
add_fake_country,
],
"setup_clean_db": [setup_clean_db],

120
apps/transfer/utils.py
View File

@ -2,6 +2,8 @@ from os.path import exists
from importlib.machinery import SourceFileLoader
from django.apps import apps
from django.conf import settings
from django.db.models import Count, Q
from tqdm import tqdm
def transfer_objects(data_type):
@ -24,6 +26,7 @@ def transfer_objects(data_type):
def clean_old_records(model, conditions):
error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w")
non_existed_correct_city_ids = []
to_delete = []
try:
old_records = model.objects.filter(**conditions)
@ -31,12 +34,14 @@ def clean_old_records(model, conditions):
error_file.write(f"Cannot find model objects: {e}")
return
for old_record in old_records:
for old_record in tqdm(old_records):
correct_record_qs = model.objects.exclude(id=old_record.id) \
.exclude(**conditions) \
.filter(name=old_record.name)
.filter(name=old_record.name,
mysql_id=old_record.old_id)
if correct_record_qs.exists():
correct_record = correct_record_qs.first()
# check object dependencies
for rel_instance in old_record._related_instances:
field_name = [related_field.name
for related_field in rel_instance._meta.fields
@ -45,14 +50,119 @@ def clean_old_records(model, conditions):
if getattr(rel_instance, field_name) != correct_record:
setattr(rel_instance, field_name, correct_record)
rel_instance.save()
to_delete.append(old_record.id)
else:
non_existed_correct_city_ids.append(old_record.id)
if non_existed_correct_city_ids:
print(f"Non existed correct city ids: {non_existed_correct_city_ids}")
return
# delete old records
counter = old_records.count()
old_records.delete()
counter = len(to_delete)
old_records.filter(id__in=to_delete).delete()
print(f'Deleted {counter} objects.')
def clean_old_country_records(model, conditions):
error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w")
non_existed_correct_country_ids = []
to_delete = []
try:
unique_codes = model.objects.values_list('code', flat=True) \
.annotate(counter=Count('code')) \
.filter(counter=1) \
.values_list('code', flat=True)
old_records = model.objects.exclude(code__in=unique_codes) \
.filter(**conditions)
except Exception as e:
error_file.write(f"Cannot find model objects: {e}")
return
for old_record in tqdm(old_records):
correct_record_qs = model.objects.exclude(id=old_record.id) \
.exclude(**conditions) \
.filter(Q(code=old_record.code) |
Q(mysql_ids__contains=[old_record.old_id, ]))
if correct_record_qs.exists():
correct_record = correct_record_qs.first()
# check object dependencies
for rel_instance in old_record._related_instances:
if not hasattr(rel_instance, '_meta'):
for related in rel_instance.all():
field_name = [related_field.name
for related_field in related._meta.fields
if related_field.related_model == correct_record._meta.model][0]
# rebinding correct dependency instances
if getattr(rel_instance, field_name) != correct_record:
setattr(rel_instance, field_name, correct_record)
rel_instance.save()
else:
field_name = [related_field.name
for related_field in rel_instance._meta.fields
if related_field.related_model == correct_record._meta.model][0]
# rebinding correct dependency instances
if getattr(rel_instance, field_name) != correct_record:
setattr(rel_instance, field_name, correct_record)
rel_instance.save()
to_delete.append(old_record.id)
else:
non_existed_correct_country_ids.append(old_record.id)
if non_existed_correct_country_ids:
print(f"Non existed correct country ids: {non_existed_correct_country_ids}")
# delete old records
counter = len(to_delete)
old_records.filter(id__in=to_delete).delete()
print(f'Deleted {counter} objects.')
def clean_old_region_records(model, conditions):
error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w")
non_existed_correct_country_ids = []
to_delete = []
try:
old_records = model.objects.filter(**conditions)
except Exception as e:
error_file.write(f"Cannot find model objects: {e}")
return
for old_record in tqdm(old_records):
correct_record_qs = model.objects.exclude(id=old_record.id) \
.exclude(**conditions) \
.filter(code__iexact=old_record.code,
mysql_ids__contains=[old_record.old_id, ])
if correct_record_qs.exists():
correct_record = correct_record_qs.first()
# check object dependencies
for rel_instance in old_record._related_instances:
if not hasattr(rel_instance, '_meta'):
for related in rel_instance.all():
field_name = [related_field.name
for related_field in related._meta.fields
if related_field.related_model == correct_record._meta.model][0]
# rebinding correct dependency instances
if getattr(rel_instance, field_name) != correct_record:
setattr(rel_instance, field_name, correct_record)
rel_instance.save()
else:
field_name = [related_field.name
for related_field in rel_instance._meta.fields
if related_field.related_model == correct_record._meta.model][0]
# rebinding correct dependency instances
if getattr(rel_instance, field_name) != correct_record:
setattr(rel_instance, field_name, correct_record)
rel_instance.save()
to_delete.append(old_record.id)
else:
non_existed_correct_country_ids.append(old_record.id)
if non_existed_correct_country_ids:
print(f"Non existed correct region ids: {non_existed_correct_country_ids}")
# delete old records
counter = len(to_delete)
old_records.filter(id__in=to_delete).delete()
print(f'Deleted {counter} objects.')

36
apps/utils/models.py
View File

@ -452,4 +452,40 @@ class FavoritesMixin:
return self.favorites.aggregate(arr=ArrayAgg('user_id')).get('arr')
class RelatedInstanceMixin:
"""Mixin for getting related objects."""
@property
def _related_objects(self) -> list:
"""Return list of related objects."""
if hasattr(self, '_meta'):
related_objects = []
for related_object in self._meta.related_objects:
related_objects.append(related_object)
return related_objects
@property
def _related_instances(self) -> set:
"""Return list of related instances."""
if hasattr(self, '_related_objects'):
related_instances = []
related_names = [related_object.related_name or f'{related_object.name}_set'
if related_object.multiple
else f'{related_object.name}'
for related_object in self._related_objects]
for related_object in related_names:
try:
instances = getattr(self, f'{related_object}')
except:
continue
if hasattr(instances, 'exists') and instances.exists():
for instance in instances.all():
related_instances.append(instance)
else:
# if one object put it in list.
related_instances.append(instances)
return set(related_instances)
timezone.datetime.now().date().isoformat()