diff --git a/.gitignore b/.gitignore index 5d44eda8..9cb2b74f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ logs/ ./docker-compose.override.yml celerybeat-schedule +local_files \ No newline at end of file diff --git a/apps/account/migrations/0018_user_old_id.py b/apps/account/migrations/0018_user_old_id.py new file mode 100644 index 00000000..aa409bed --- /dev/null +++ b/apps/account/migrations/0018_user_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-29 08:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('account', '0017_merge_20191024_1233'), + ] + + operations = [ + migrations.AddField( + model_name='user', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/account/models.py b/apps/account/models.py index 2f8c97cc..3eb07722 100644 --- a/apps/account/models.py +++ b/apps/account/models.py @@ -94,6 +94,8 @@ class User(AbstractUser): unconfirmed_email = models.EmailField(_('unconfirmed email'), blank=True, null=True, default=None) email_confirmed = models.BooleanField(_('email status'), default=False) newsletter = models.NullBooleanField(default=True) + old_id = models.IntegerField(null=True, blank=True, default=None) + EMAIL_FIELD = 'email' USERNAME_FIELD = 'username' @@ -241,6 +243,20 @@ class User(AbstractUser): template_name=settings.CHANGE_EMAIL_TEMPLATE, context=context) + @property + def favorite_establishment_ids(self): + """Return establishment IDs that in favorites for current user.""" + return self.favorites.by_content_type(app_label='establishment', + model='establishment')\ + .values_list('object_id', flat=True) + + @property + def favorite_recipe_ids(self): + """Return recipe IDs that in favorites for current user.""" + return self.favorites.by_content_type(app_label='recipe', + model='recipe')\ + .values_list('object_id', flat=True) + class UserRole(ProjectBaseMixin): """UserRole model.""" diff --git a/apps/account/transfer.py b/apps/account/transfer.py new file mode 100644 index 00000000..54db107b --- /dev/null +++ b/apps/account/transfer.py @@ -0,0 +1,53 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + +""" + +card = { + "User": { + "data_type": "objects", + "dependencies": None, + "fields": { + "Accounts": { + "username": "nickname", + "email": "email", + "email_confirmed": ("confirmed_at", "django.db.models.BooleanField") + }, + "relations": { + "Profiles": { + "key": "account", + "fields": { + "first_name": "firstname", + "last_name": "lastname" + } + } + } + } + } + +} + +used_apps = None + + diff --git a/apps/account/transfer_data.py b/apps/account/transfer_data.py new file mode 100644 index 00000000..c39e0ac1 --- /dev/null +++ b/apps/account/transfer_data.py @@ -0,0 +1,28 @@ +from django.db.models import Value, IntegerField, F +from pprint import pprint +from transfer.models import Profiles, Accounts +from transfer.serializers.account import UserSerializer + + +def transfer_user(): + # queryset = Profiles.objects.all() + # queryset = queryset.annotate(nickname=F("account__nickname")) + # queryset = queryset.annotate(email=F("account__email")) + stop_list = ['cyril@tomatic.net', + 'cyril2@tomatic.net', + 'd.sadykova@id-east.ru', + 'd.sadykova@octopod.ru', + 'n.yurchenko@id-east.ru'] + queryset = Accounts.objects.filter(confirmed_at__isnull=False).exclude(email__in=stop_list) + + serialized_data = UserSerializer(data=list(queryset.values()), many=True) + + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "account": [transfer_user] +} diff --git a/apps/advertisement/migrations/0004_auto_20191025_0903.py b/apps/advertisement/migrations/0004_auto_20191025_0903.py new file mode 100644 index 00000000..f633d67c --- /dev/null +++ b/apps/advertisement/migrations/0004_auto_20191025_0903.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-25 09:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('advertisement', '0003_auto_20190919_1344'), + ] + + operations = [ + migrations.AlterField( + model_name='advertisement', + name='block_level', + field=models.CharField(blank=True, max_length=10, null=True, verbose_name='Block level'), + ), + ] diff --git a/apps/advertisement/models.py b/apps/advertisement/models.py index fdc25988..62b41428 100644 --- a/apps/advertisement/models.py +++ b/apps/advertisement/models.py @@ -15,7 +15,7 @@ class Advertisement(ImageMixin, ProjectBaseMixin, PlatformMixin): url = models.URLField(verbose_name=_('Ad URL')) width = models.PositiveIntegerField(verbose_name=_('Block width')) height = models.PositiveIntegerField(verbose_name=_('Block height')) - block_level = models.CharField(verbose_name=_('Block level'), max_length=10) + block_level = models.CharField(verbose_name=_('Block level'), max_length=10, blank=True, null=True) target_languages = models.ManyToManyField(Language) class Meta: diff --git a/apps/advertisement/transfer_data.py b/apps/advertisement/transfer_data.py new file mode 100644 index 00000000..3af16286 --- /dev/null +++ b/apps/advertisement/transfer_data.py @@ -0,0 +1,20 @@ +from pprint import pprint +from django.db.models import Value, IntegerField, F +from transfer.models import Ads +from transfer.serializers.advertisement import AdvertisementSerializer + + +def transfer_advertisement(): + queryset = Ads.objects.filter(href__isnull=False) + + serialized_data = AdvertisementSerializer(data=list(queryset.values()), many=True) + + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "commercial": [transfer_advertisement] +} \ No newline at end of file diff --git a/apps/collection/transfer.py b/apps/collection/transfer.py new file mode 100644 index 00000000..6c18c9ae --- /dev/null +++ b/apps/collection/transfer.py @@ -0,0 +1,48 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +""" +card = { + "Collection": { + "data_type": "objects", + "dependencies": ("Country", ), + "fields": { + "Collections": { + # нету аналогов для полей description, start и end + "name": "title", + "slug": "slug", + "block_size": ("geometries", "django.db.models.JSONField"), + "is_publish": ("active", "django.db.models.BooleanField"), + "image_url": ("attachment_file_name", "django.db.models.URLField") + } + }, + "relations": { + # "country": "Country", + } + }, + "Guide": { + # как работать с ForeignKey на самого себя(self), поле "parent" + "data_type": "objects", + "dependencies": ("Collection", "self"), + "fields": { + "Guides": { + # нету аналогов для полей start и end + "name": "title" + } + }, + "relations": { + # аналалог для поля "collection" не найдено + # "parent": "Guide", + # "collection": "Collection" + } + } +} + +used_apps = ("location", ) diff --git a/apps/comment/migrations/0004_comment_old_id.py b/apps/comment/migrations/0004_comment_old_id.py new file mode 100644 index 00000000..46dbb7d0 --- /dev/null +++ b/apps/comment/migrations/0004_comment_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-29 12:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('comment', '0003_auto_20191015_0704'), + ] + + operations = [ + migrations.AddField( + model_name='comment', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/comment/models.py b/apps/comment/models.py index 55c7802e..29021677 100644 --- a/apps/comment/models.py +++ b/apps/comment/models.py @@ -45,6 +45,8 @@ class Comment(ProjectBaseMixin): objects = CommentQuerySet.as_manager() country = models.ForeignKey(Country, verbose_name=_('Country'), on_delete=models.SET_NULL, null=True) + old_id = models.IntegerField(null=True, blank=True, default=None) + class Meta: """Meta class""" diff --git a/apps/comment/transfer.py b/apps/comment/transfer.py new file mode 100644 index 00000000..64c0abab --- /dev/null +++ b/apps/comment/transfer.py @@ -0,0 +1,49 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + +""" + +card = { + # как работать с GenericForeignKey(content_type) - ? + "Comment": { + "data_type": "objects", + "dependencies": ("User",), + "fields": { + "Comments": { + "text": "comment", + "mark": ("mark", "django.db.models.PositiveIntegerField") + # как работать с GenericForeignKey - ? + # "content_object" : "" + }, + }, + "relations": { + "Accounts": [ + (("account", None), + ("User", "user", None, None)) + ] + } + } +} + +used_apps = ("account", ) \ No newline at end of file diff --git a/apps/comment/transfer_data.py b/apps/comment/transfer_data.py new file mode 100644 index 00000000..3a291b86 --- /dev/null +++ b/apps/comment/transfer_data.py @@ -0,0 +1,21 @@ +from pprint import pprint +from transfer.models import Comments +from transfer.serializers.comments import CommentSerializer + + +def transfer_comments(): + queryset = Comments.objects.filter(account__isnull=False, mark__isnull=False)\ + .only("id", "comment", "mark", "locale", "establishment_id", "account_id") + + serialized_data = CommentSerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(serialized_data.errors) + + +data_types = { + "tmp": [ + # transfer_comments + ] +} diff --git a/apps/establishment/admin.py b/apps/establishment/admin.py index 66f0aee1..62b8a8da 100644 --- a/apps/establishment/admin.py +++ b/apps/establishment/admin.py @@ -4,6 +4,7 @@ from django.contrib.contenttypes.admin import GenericTabularInline from django.utils.translation import gettext_lazy as _ from comment.models import Comment +from utils.admin import BaseModelAdminMixin from establishment import models from main.models import Award from product.models import Product @@ -11,12 +12,12 @@ from review import models as review_models @admin.register(models.EstablishmentType) -class EstablishmentTypeAdmin(admin.ModelAdmin): +class EstablishmentTypeAdmin(BaseModelAdminMixin, admin.ModelAdmin): """EstablishmentType admin.""" @admin.register(models.EstablishmentSubType) -class EstablishmentSubTypeAdmin(admin.ModelAdmin): +class EstablishmentSubTypeAdmin(BaseModelAdminMixin, admin.ModelAdmin): """EstablishmentSubType admin.""" @@ -37,7 +38,7 @@ class ContactEmailInline(admin.TabularInline): extra = 0 -class ReviewInline(GenericTabularInline): +class ReviewInline(BaseModelAdminMixin, GenericTabularInline): model = review_models.Review extra = 0 @@ -53,16 +54,17 @@ class ProductInline(admin.TabularInline): @admin.register(models.Establishment) -class EstablishmentAdmin(admin.ModelAdmin): +class EstablishmentAdmin(BaseModelAdminMixin, admin.ModelAdmin): """Establishment admin.""" list_display = ['id', '__str__', 'image_tag', ] inlines = [ AwardInline, ContactPhoneInline, ContactEmailInline, ReviewInline, CommentInline, ProductInline] + raw_id_fields = ('address',) @admin.register(models.Position) -class PositionAdmin(admin.ModelAdmin): +class PositionAdmin(BaseModelAdminMixin, admin.ModelAdmin): """Position admin.""" @@ -73,7 +75,7 @@ class PlateInline(admin.TabularInline): @admin.register(models.Menu) -class MenuAdmin(admin.ModelAdmin): +class MenuAdmin(BaseModelAdminMixin, admin.ModelAdmin): """Menu admin.""" list_display = ['id', 'category_translated'] inlines = [ diff --git a/apps/establishment/management/commands/add_establishment_description.py b/apps/establishment/management/commands/add_establishment_description.py new file mode 100644 index 00000000..1748d81d --- /dev/null +++ b/apps/establishment/management/commands/add_establishment_description.py @@ -0,0 +1,31 @@ +from django.core.management.base import BaseCommand + +from establishment.models import Establishment +from transfer.models import Descriptions + + +class Command(BaseCommand): + help = 'Add description values from old db to new db' + + def handle(self, *args, **kwargs): + count = 0 + + queryset = Descriptions.objects.all() + for obj in queryset: + try: + establishment = Establishment.objects.get(old_id=obj.establishment.id) + except Establishment.DoesNotExist: + continue + except Establishment.MultipleObjectsReturned: + establishment = Establishment.objects.filter(old_id=obj.establishment.id).first() + else: + description = establishment.description + description.update({ + obj.locale: obj.text + }) + establishment.description = description + establishment.save() + count += 1 + break + + self.stdout.write(self.style.WARNING(f'Updated {count} objects.')) diff --git a/apps/establishment/management/commands/add_establishment_image.py b/apps/establishment/management/commands/add_establishment_image.py new file mode 100644 index 00000000..3e88a398 --- /dev/null +++ b/apps/establishment/management/commands/add_establishment_image.py @@ -0,0 +1,32 @@ +from django.core.management.base import BaseCommand + +from establishment.models import Establishment +from transfer.models import EstablishmentAssets + + +class Command(BaseCommand): + help = 'Add preview_image_url values from old db to new db' + + def handle(self, *args, **kwargs): + count = 0 + url = 'https://1dc3f33f6d-3.optimicdn.com/gaultmillau.com/' + raw_qs = EstablishmentAssets.objects.raw( + '''SELECT establishment_assets.id, establishment_id, attachment_suffix_url + FROM establishment_assets + WHERE attachment_file_name IS NOT NULL + AND attachment_suffix_url IS NOT NULL + AND scope = 'public' + AND type = 'Photo' + AND menu_id IS NULL + GROUP BY establishment_id;''' + ) + queryset = [vars(query) for query in raw_qs] + for obj in queryset: + establishment = Establishment.objects.filter(old_id=obj['establishment_id']).first() + if establishment: + img = url + obj['attachment_suffix_url'] + establishment.preview_image_url = img + establishment.image_url = img + establishment.save() + count += 1 + self.stdout.write(self.style.WARNING(f'Updated {count} objects.')) diff --git a/apps/establishment/management/commands/add_establishment_mark.py b/apps/establishment/management/commands/add_establishment_mark.py new file mode 100644 index 00000000..1588268f --- /dev/null +++ b/apps/establishment/management/commands/add_establishment_mark.py @@ -0,0 +1,46 @@ +from pprint import pprint + +from django.core.management.base import BaseCommand +from django.db.models import Q + +from establishment.models import Establishment +from transfer.models import Reviews + + +class Command(BaseCommand): + help = 'Add description values from old db to new db' + + def handle(self, *args, **kwargs): + count = 0 + valid_data = {} + + queryset = Reviews.objects.exclude( + Q(establishment_id__isnull=True) | + Q(mark__isnull=True) + ).filter(aasm_state='published').values_list('establishment_id', 'mark', 'updated_at') + + print(queryset.count()) + + for es_id, new_mark, new_date in queryset: + try: + mark, date = valid_data[es_id] + except KeyError: + valid_data[es_id] = (new_mark, new_date) + else: + if new_date > date: + valid_data[es_id] = (new_mark, new_date) + + for key, value in valid_data.items(): + try: + establishment = Establishment.objects.get(old_id=key) + except Establishment.DoesNotExist: + continue + except Establishment.MultipleObjectsReturned: + establishment = Establishment.objects.filter(old_id=key).first() + else: + establishment.public_mark = int(value[0]) + establishment.save() + count += 1 + break + + self.stdout.write(self.style.WARNING(f'Updated {count} objects.')) diff --git a/apps/establishment/management/commands/add_publish_data.py b/apps/establishment/management/commands/add_publish_data.py new file mode 100644 index 00000000..81a52890 --- /dev/null +++ b/apps/establishment/management/commands/add_publish_data.py @@ -0,0 +1,19 @@ +from django.core.management.base import BaseCommand + +from establishment.models import Establishment +from transfer.models import Establishments + + +class Command(BaseCommand): + help = 'Add publish values from old db to new db' + + def handle(self, *args, **kwargs): + old_establishments = Establishments.objects.all() + count = 0 + for item in old_establishments: + obj = Establishment.objects.filter(old_id=item.id).first() + if obj: + count += 1 + obj.is_publish = item.state == 'published' + obj.save() + self.stdout.write(self.style.WARNING(f'Updated {count} objects.')) diff --git a/apps/establishment/management/commands/rm_old_establishments.py b/apps/establishment/management/commands/rm_old_establishments.py new file mode 100644 index 00000000..ec9fbbaa --- /dev/null +++ b/apps/establishment/management/commands/rm_old_establishments.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand + +from establishment.models import Establishment + + +class Command(BaseCommand): + help = 'Remove old establishments from new bd' + + def handle(self, *args, **kwargs): + old_establishments = Establishment.objects.exclude(old_id__isnull=True) + count = old_establishments.count() + old_establishments.delete() + self.stdout.write(self.style.WARNING(f'Deleted {count} objects.')) diff --git a/apps/establishment/migrations/0044_establishment_old_id.py b/apps/establishment/migrations/0044_establishment_old_id.py new file mode 100644 index 00000000..9f630c2b --- /dev/null +++ b/apps/establishment/migrations/0044_establishment_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-27 07:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('establishment', '0043_establishment_currency'), + ] + + operations = [ + migrations.AddField( + model_name='establishment', + name='old_id', + field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id'), + ), + ] diff --git a/apps/establishment/migrations/0045_auto_20191029_0719.py b/apps/establishment/migrations/0045_auto_20191029_0719.py new file mode 100644 index 00000000..4ceb137b --- /dev/null +++ b/apps/establishment/migrations/0045_auto_20191029_0719.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2.4 on 2019-10-29 07:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('establishment', '0044_establishment_old_id'), + ] + + operations = [ + migrations.AlterField( + model_name='establishment', + name='booking', + field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Booking URL'), + ), + migrations.AlterField( + model_name='establishment', + name='facebook', + field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Facebook URL'), + ), + migrations.AlterField( + model_name='establishment', + name='lafourchette', + field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Lafourchette URL'), + ), + migrations.AlterField( + model_name='establishment', + name='preview_image_url', + field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Preview image URL path'), + ), + migrations.AlterField( + model_name='establishment', + name='twitter', + field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Twitter URL'), + ), + migrations.AlterField( + model_name='establishment', + name='website', + field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Web site URL'), + ), + ] diff --git a/apps/establishment/migrations/0046_merge_20191030_0858.py b/apps/establishment/migrations/0046_merge_20191030_0858.py new file mode 100644 index 00000000..39153492 --- /dev/null +++ b/apps/establishment/migrations/0046_merge_20191030_0858.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 08:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('establishment', '0045_auto_20191029_0719'), + ('establishment', '0044_position_index_name'), + ] + + operations = [ + ] diff --git a/apps/establishment/migrations/0046_merge_20191030_1618.py b/apps/establishment/migrations/0046_merge_20191030_1618.py new file mode 100644 index 00000000..853b25eb --- /dev/null +++ b/apps/establishment/migrations/0046_merge_20191030_1618.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 16:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('establishment', '0045_auto_20191029_0719'), + ('establishment', '0044_position_index_name'), + ] + + operations = [ + ] diff --git a/apps/establishment/migrations/0047_merge_20191030_1714.py b/apps/establishment/migrations/0047_merge_20191030_1714.py new file mode 100644 index 00000000..845a472e --- /dev/null +++ b/apps/establishment/migrations/0047_merge_20191030_1714.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 17:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('establishment', '0046_merge_20191030_1618'), + ('establishment', '0046_merge_20191030_0858'), + ] + + operations = [ + ] diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 0ca2463f..981f9bb3 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -14,6 +14,8 @@ from django.db.models import When, Case, F, ExpressionWrapper, Subquery, Q from django.utils import timezone from django.utils.translation import gettext_lazy as _ from phonenumber_field.modelfields import PhoneNumberField +from pytz import timezone as ptz +from timezone_field import TimeZoneField from collection.models import Collection from location.models import Address @@ -21,7 +23,6 @@ from main.models import Award, Currency from review.models import Review from utils.models import (ProjectBaseMixin, TJSONField, URLImageMixin, TranslatedFieldsMixin, BaseAttributes) -from timezone_field import TimeZoneField # todo: establishment type&subtypes check @@ -248,14 +249,12 @@ class EstablishmentQuerySet(models.QuerySet): def annotate_in_favorites(self, user): """Annotate flag in_favorites""" - favorite_establishments = [] + favorite_establishment_ids = [] if user.is_authenticated: - favorite_establishments = user.favorites.by_content_type(app_label='establishment', - model='establishment') \ - .values_list('object_id', flat=True) + favorite_establishment_ids = user.favorite_establishment_ids return self.annotate(in_favorites=Case( When( - id__in=favorite_establishments, + id__in=favorite_establishment_ids, then=True), default=False, output_field=models.BooleanField(default=False))) @@ -301,6 +300,7 @@ class EstablishmentQuerySet(models.QuerySet): class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin): """Establishment model.""" + old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None) name = models.CharField(_('name'), max_length=255, default='') name_translated = models.CharField(_('Transliterated name'), max_length=255, default='') @@ -327,19 +327,19 @@ class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin): price_level = models.PositiveIntegerField(blank=True, null=True, default=None, verbose_name=_('price level')) - website = models.URLField(blank=True, null=True, default=None, + website = models.URLField(blank=True, null=True, default=None, max_length=255, verbose_name=_('Web site URL')) - facebook = models.URLField(blank=True, null=True, default=None, + facebook = models.URLField(blank=True, null=True, default=None, max_length=255, verbose_name=_('Facebook URL')) - twitter = models.URLField(blank=True, null=True, default=None, + twitter = models.URLField(blank=True, null=True, default=None, max_length=255, verbose_name=_('Twitter URL')) - lafourchette = models.URLField(blank=True, null=True, default=None, + lafourchette = models.URLField(blank=True, null=True, default=None, max_length=255, verbose_name=_('Lafourchette URL')) guestonline_id = models.PositiveIntegerField(blank=True, verbose_name=_('guestonline id'), null=True, default=None,) lastable_id = models.TextField(blank=True, verbose_name=_('lastable id'), unique=True, null=True, default=None,) - booking = models.URLField(blank=True, null=True, default=None, + booking = models.URLField(blank=True, null=True, default=None, max_length=255, verbose_name=_('Booking URL')) is_publish = models.BooleanField(default=False, verbose_name=_('Publish status')) schedule = models.ManyToManyField(to='timetable.Timetable', @@ -355,7 +355,7 @@ class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin): related_name='establishments', blank=True, default=None, verbose_name=_('Collections')) - preview_image_url = models.URLField(verbose_name=_('Preview image URL path'), + preview_image_url = models.URLField(verbose_name=_('Preview image URL path'), max_length=255, blank=True, null=True, default=None) slug = models.SlugField(unique=True, max_length=255, null=True, verbose_name=_('Establishment slug')) diff --git a/apps/establishment/serializers/common.py b/apps/establishment/serializers/common.py index 14be142a..3f42c15b 100644 --- a/apps/establishment/serializers/common.py +++ b/apps/establishment/serializers/common.py @@ -5,15 +5,14 @@ from rest_framework import serializers from comment import models as comment_models from comment.serializers import common as comment_serializers from establishment import models -from favorites.models import Favorites from location.serializers import AddressBaseSerializer from main.serializers import AwardSerializer, CurrencySerializer from review import models as review_models from tag.serializers import TagBaseSerializer from timetable.serialziers import ScheduleRUDSerializer from utils import exceptions as utils_exceptions -from utils.serializers import ProjectModelSerializer -from utils.serializers import TranslatedField +from utils.serializers import (ProjectModelSerializer, TranslatedField, + FavoritesCreateSerializer) class ContactPhonesSerializer(serializers.ModelSerializer): @@ -118,6 +117,18 @@ class EstablishmentTypeBaseSerializer(serializers.ModelSerializer): 'use_subtypes': {'write_only': True}, } +class EstablishmentTypeGeoSerializer(EstablishmentTypeBaseSerializer): + """Serializer for EstablishmentType model w/ index_name.""" + + class Meta(EstablishmentTypeBaseSerializer.Meta): + fields = EstablishmentTypeBaseSerializer.Meta.fields + [ + 'index_name' + ] + extra_kwargs = { + **EstablishmentTypeBaseSerializer.Meta.extra_kwargs, + 'index_name': {'read_only': True}, + } + class EstablishmentSubTypeBaseSerializer(serializers.ModelSerializer): """Serializer for EstablishmentSubType models.""" @@ -185,6 +196,19 @@ class EstablishmentBaseSerializer(ProjectModelSerializer): ] +class EstablishmentGeoSerializer(EstablishmentBaseSerializer): + """Serializer for Geo view.""" + + type = EstablishmentTypeGeoSerializer(source='establishment_type', read_only=True) + + class Meta(EstablishmentBaseSerializer.Meta): + """Meta class.""" + + fields = EstablishmentBaseSerializer.Meta.fields + [ + 'type' + ] + + class EstablishmentDetailSerializer(EstablishmentBaseSerializer): """Serializer for Establishment model.""" @@ -281,26 +305,13 @@ class EstablishmentCommentRUDSerializer(comment_serializers.CommentSerializer): ] -class EstablishmentFavoritesCreateSerializer(serializers.ModelSerializer): - """Create comment serializer""" - - class Meta: - """Serializer for model Comment""" - model = Favorites - fields = [ - 'id', - 'created', - ] - - def get_user(self): - """Get user from request""" - return self.context.get('request').user +class EstablishmentFavoritesCreateSerializer(FavoritesCreateSerializer): + """Serializer to favorite object w/ model Establishment.""" def validate(self, attrs): - """Override validate method""" + """Overridden validate method""" # Check establishment object - establishment_slug = self.context.get('request').parser_context.get('kwargs').get('slug') - establishment_qs = models.Establishment.objects.filter(slug=establishment_slug) + establishment_qs = models.Establishment.objects.filter(slug=self.slug) # Check establishment obj by slug from lookup_kwarg if not establishment_qs.exists(): @@ -309,18 +320,16 @@ class EstablishmentFavoritesCreateSerializer(serializers.ModelSerializer): establishment = establishment_qs.first() # Check existence in favorites - if self.get_user().favorites.by_content_type(app_label='establishment', - model='establishment')\ - .by_object_id(object_id=establishment.id).exists(): + if establishment.favorites.filter(user=self.user).exists(): raise utils_exceptions.FavoritesError() attrs['establishment'] = establishment return attrs def create(self, validated_data, *args, **kwargs): - """Override create method""" + """Overridden create method""" validated_data.update({ - 'user': self.get_user(), + 'user': self.user, 'content_object': validated_data.pop('establishment') }) return super().create(validated_data) diff --git a/apps/establishment/transfer.py b/apps/establishment/transfer.py new file mode 100644 index 00000000..677245f9 --- /dev/null +++ b/apps/establishment/transfer.py @@ -0,0 +1,119 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +""" + +card = { + "EstablishmentType": { + "data_type": "dictionaries", + "dependencies": None, + "fields": { + "Establishments": { + # значения для поля "name" берутся из поля "type" legacy модели "establishments" + # https://jira.spider.ru/browse/GM-199 + "name": ("name", "django.db.models.TJSONField") + } + } + }, + "Establishment": { + "data_type": "dictionaries", + "dependencies": ("Address", "Collection", "EstablishmentType"), + # нету аналогов для establishment_type, establishment_subtypes, schedule, comments, tags + "fields": { + "Establishments": { + "name": "name", + "slug": "slug", + "is_publish": ("state", "django.db.models.BooleanField") + }, + "relations": { + "EstablishmentInfos": { + "key": "establishment", + "fields": { + "website": "website", + "facebook": "facebook", + "twitter": "twitter", + "lafourchette": "lafourchette", + "booking": "booking_url" + } + }, + }, + }, + # как работать с GenericRelation - ? + # как работать с ManyToManyField - ? "EstablishmentSubType", "schedule" + "relations": { + "Locations": [ + (("location", None), + ("Address", "address", None, None)), + ], + "Establishments": [ # TODO правильно ли заполнена связь с EstablishmentType - ? + (("type", "type"), + ("EstablishmentType", "establishment_type", "id", "django.db.models.PositiveIntegerField")) + ] + # # "establishment_subtypes": "EstablishmentSubType", + # "collections": "Collection", + # # TODO: нашел schedules в legacy + # # "schedule": "Timetable", + # "award": "Award", + # # "tags": "MetaDataContent", + # "reviews": "Review", + # # "comments": "Comment", + # # "favorites": "Favorites" # В legacy этой таблицы не было + + } + }, + "Menu": { + "data_type": "objects", + "dependencies": ("Establishment",), + "fields": { + "Menus": { + "category": ("name", "django.db.models.TJSONField") + } + }, + "relations": { + "Establishments": [ + (("establishment", None), + ("Establishment", "establishment", None, None)) + ] + } + }, + "ContactPhone": { + "data_type": "objects", + "dependencies": ("Establishment",), + "fields": { + "Establishments": { + "phone": "phone" + }, + "relations": { # TODO правильно ли заполнена связь с Establishment - ? + "Establishments": [ + (("id", "id"), + ("Establishment", "establishment", "id", "django.db.models.PositiveIntegerField")) + ] + } + } + }, + + "ContactEmail": { + "data_type": "objects", + "dependencies": ("Establishment",), + "fields": { + "EstablishmentInfos": { + "email": "email" + }, + }, + "relations": { + "Establishments": [ + (("establishment", None), + ("Establishment", "establishment", None, None)) + ] + } + } +} + +used_apps = ("review", "location", "collection", "main", "timetable", "favorites", "comment",) diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py new file mode 100644 index 00000000..8366ed94 --- /dev/null +++ b/apps/establishment/transfer_data.py @@ -0,0 +1,138 @@ +from pprint import pprint + +from django.db.models import Q, F + +from transfer.models import Establishments, Dishes +from transfer.serializers.establishment import EstablishmentSerializer +from establishment.models import Establishment +from location.models import Address +from transfer.serializers.plate import PlateSerializer + + +def transfer_establishment(): + result = [] + + old_establishments = Establishments.objects.filter( + location__city__name__icontains='paris', + ).exclude( + Q(type='Wineyard') | + Q(location__timezone__isnull=True) + ).prefetch_related( + 'establishmentinfos_set', + 'schedules_set', + 'descriptions_set', + ) + + for item in old_establishments: + data = { + 'old_id': item.id, + 'name': item.name, + 'name_translated': item.index_name, + 'slug': item.slug, + 'type': item.type, + 'phone': item.phone, + 'created': item.created_at, + 'state': item.state, + 'description': {}, + 'website': None, + 'facebook': None, + 'twitter': None, + 'lafourchette': None, + 'booking': None, + 'schedules': None, + 'email': None, + } + + if item.location: + data.update({ + 'location': item.location.id, + 'tz': item.location.timezone, + }) + + # Инфо + info = item.establishmentinfos_set.first() + if info: + data.update({ + 'website': info.website, + 'facebook': info.facebook, + 'twitter': info.twitter, + 'lafourchette': info.lafourchette, + 'booking': info.booking_url, + 'email': info.email, + }) + + # Время работы + schedule = item.schedules_set.first() + if schedule: + data.update({ + 'schedules': schedule.timetable, + }) + + # Описание + descriptions = item.descriptions_set.all() + for description in descriptions: + data['description'].update({ + description.locale: description.text, + }) + + result.append(data) + + serialized_data = EstablishmentSerializer(data=result, many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"Establishment serializer errors: {serialized_data.errors}") + + +def transfer_menu(): + dishes = Dishes.objects.exclude( + Q(establishment_id__isnull=True) | + Q(dish_type__isnull=True) | + Q(price__isnull=True) | + Q(currency__isnull=True) | + Q(name__isnull=True) | + Q(name__exact='') + ).annotate( + country_code=F('establishment__location__country_code'), + ) + + serialized_data = PlateSerializer(data=list(dishes.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"Menu serializer errors: {serialized_data.errors}") + + +def transfer_establishment_addresses(): + old_establishments = Establishments.objects.only("id", "location_id").exclude( + Q(type='Wineyard') | + Q(location__timezone__isnull=True) + ) + + for old_establishment in old_establishments: + try: + establishment = Establishment.objects.get(old_id=old_establishment.id) + except Establishment.DoesNotExist: + continue + + try: + location = Address.objects.get(old_id=old_establishment.location_id) + except Address.MultipleObjectsReturned: + location = Address.objects.filter(old_id=old_establishment.location_id).first() + except Address.DoesNotExist: + continue + + establishment.address = location + establishment.save() + + +data_types = { + "establishment": [ + transfer_establishment, + ], + "location_establishment": [ + transfer_establishment_addresses + ], + + "menu": [transfer_menu], +} diff --git a/apps/establishment/urls/common.py b/apps/establishment/urls/common.py index 8d9453c1..49cd3631 100644 --- a/apps/establishment/urls/common.py +++ b/apps/establishment/urls/common.py @@ -9,7 +9,6 @@ urlpatterns = [ path('', views.EstablishmentListView.as_view(), name='list'), path('recent-reviews/', views.EstablishmentRecentReviewListView.as_view(), name='recent-reviews'), - # path('wineries/', views.WineriesListView.as_view(), name='wineries-list'), path('slug//', views.EstablishmentRetrieveView.as_view(), name='detail'), path('slug//similar/', views.EstablishmentSimilarListView.as_view(), name='similar'), path('slug//comments/', views.EstablishmentCommentListView.as_view(), name='list-comments'), @@ -18,5 +17,5 @@ urlpatterns = [ path('slug//comments//', views.EstablishmentCommentRUDView.as_view(), name='rud-comment'), path('slug//favorites/', views.EstablishmentFavoritesCreateDestroyView.as_view(), - name='add-to-favorites') + name='create-destroy-favorites') ] diff --git a/apps/establishment/views/web.py b/apps/establishment/views/web.py index 0699d9d0..05391827 100644 --- a/apps/establishment/views/web.py +++ b/apps/establishment/views/web.py @@ -138,21 +138,18 @@ class EstablishmentFavoritesCreateDestroyView(generics.CreateAPIView, generics.D """ Returns the object the view is displaying. """ - establishment_obj = get_object_or_404(models.Establishment, - slug=self.kwargs['slug']) - obj = get_object_or_404( - self.request.user.favorites.by_content_type(app_label='establishment', - model='establishment') - .by_object_id(object_id=establishment_obj.pk)) + establishment = get_object_or_404(models.Establishment, + slug=self.kwargs['slug']) + favorites = get_object_or_404(establishment.favorites.filter(user=self.request.user)) # May raise a permission denied - self.check_object_permissions(self.request, obj) - return obj + self.check_object_permissions(self.request, favorites) + return favorites class EstablishmentNearestRetrieveView(EstablishmentListView, generics.ListAPIView): """Resource for getting list of nearest establishments.""" - serializer_class = serializers.EstablishmentBaseSerializer + serializer_class = serializers.EstablishmentGeoSerializer filter_class = filters.EstablishmentFilter def get_queryset(self): @@ -170,14 +167,3 @@ class EstablishmentNearestRetrieveView(EstablishmentListView, generics.ListAPIVi return qs.by_distance_from_point(**{k: v for k, v in filter_kwargs.items() if v is not None}) return qs - - -# Wineries -# todo: find out about difference between subtypes data -# class WineriesListView(EstablishmentListView): -# """Return list establishments with type Wineries""" -# -# def get_queryset(self): -# """Overridden get_queryset method.""" -# qs = super(WineriesListView, self).get_queryset() -# return qs.with_type_related().wineries() diff --git a/apps/favorites/urls.py b/apps/favorites/urls.py index bd0c1d16..ad4c6e9d 100644 --- a/apps/favorites/urls.py +++ b/apps/favorites/urls.py @@ -8,4 +8,6 @@ app_name = 'favorites' urlpatterns = [ path('establishments/', views.FavoritesEstablishmentListView.as_view(), name='establishment-list'), + path('products/', views.FavoritesProductListView.as_view(), + name='product-list'), ] diff --git a/apps/favorites/views.py b/apps/favorites/views.py index 5d99ed4b..d2973142 100644 --- a/apps/favorites/views.py +++ b/apps/favorites/views.py @@ -1,7 +1,11 @@ """Views for app favorites.""" from rest_framework import generics from establishment.models import Establishment +from establishment.filters import EstablishmentFilter from establishment.serializers import EstablishmentBaseSerializer +from product.models import Product +from product.serializers import ProductBaseSerializer +from product.filters import ProductFilterSet from .models import Favorites @@ -14,11 +18,24 @@ class FavoritesBaseView(generics.GenericAPIView): class FavoritesEstablishmentListView(generics.ListAPIView): - """List views for favorites""" + """List views for establishments in favorites.""" serializer_class = EstablishmentBaseSerializer + filter_class = EstablishmentFilter def get_queryset(self): """Override get_queryset method""" return Establishment.objects.filter(favorites__user=self.request.user)\ .order_by('-favorites') + + +class FavoritesProductListView(generics.ListAPIView): + """List views for products in favorites.""" + + serializer_class = ProductBaseSerializer + filter_class = ProductFilterSet + + def get_queryset(self): + """Override get_queryset method""" + return Product.objects.filter(favorites__user=self.request.user)\ + .order_by('-favorites') diff --git a/apps/gallery/migrations/0002_auto_20191023_1207.py b/apps/gallery/migrations/0002_auto_20191023_1207.py new file mode 100644 index 00000000..64b93720 --- /dev/null +++ b/apps/gallery/migrations/0002_auto_20191023_1207.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.4 on 2019-10-23 12:07 + +from django.db import migrations +import easy_thumbnails.fields +import utils.methods + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='image', + field=easy_thumbnails.fields.ThumbnailerImageField(max_length=255, upload_to=utils.methods.image_path, verbose_name='Image file'), + ), + ] diff --git a/apps/gallery/migrations/0004_merge_20191025_0906.py b/apps/gallery/migrations/0004_merge_20191025_0906.py new file mode 100644 index 00000000..4372fa95 --- /dev/null +++ b/apps/gallery/migrations/0004_merge_20191025_0906.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-25 09:06 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0002_auto_20191023_1207'), + ('gallery', '0003_auto_20191003_1228'), + ] + + operations = [ + ] diff --git a/apps/gallery/migrations/0005_auto_20191025_0914.py b/apps/gallery/migrations/0005_auto_20191025_0914.py new file mode 100644 index 00000000..61c158b6 --- /dev/null +++ b/apps/gallery/migrations/0005_auto_20191025_0914.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.4 on 2019-10-25 09:14 + +from django.db import migrations +import sorl.thumbnail.fields +import utils.methods + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0004_merge_20191025_0906'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='image', + field=sorl.thumbnail.fields.ImageField(upload_to=utils.methods.image_path, verbose_name='image file'), + ), + ] diff --git a/apps/gallery/migrations/0005_auto_20191027_0756.py b/apps/gallery/migrations/0005_auto_20191027_0756.py new file mode 100644 index 00000000..d60745f1 --- /dev/null +++ b/apps/gallery/migrations/0005_auto_20191027_0756.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.4 on 2019-10-27 07:56 + +from django.db import migrations +import sorl.thumbnail.fields +import utils.methods + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0004_merge_20191025_0906'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='image', + field=sorl.thumbnail.fields.ImageField(upload_to=utils.methods.image_path, verbose_name='image file'), + ), + ] diff --git a/apps/gallery/migrations/0006_merge_20191027_1758.py b/apps/gallery/migrations/0006_merge_20191027_1758.py new file mode 100644 index 00000000..5fca7c74 --- /dev/null +++ b/apps/gallery/migrations/0006_merge_20191027_1758.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-27 17:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0005_auto_20191027_0756'), + ('gallery', '0005_auto_20191025_0914'), + ] + + operations = [ + ] diff --git a/apps/gallery/transfer.py b/apps/gallery/transfer.py new file mode 100644 index 00000000..50d99830 --- /dev/null +++ b/apps/gallery/transfer.py @@ -0,0 +1,26 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +""" + +card = { + "Image": { + "data_type": "objects", + "dependencies": None, + "fields": { + "MercuryImages": { + "image": "attachment_file_name" + } + } + }, + +} + +used_apps = None diff --git a/apps/gallery/transfer_data.py b/apps/gallery/transfer_data.py new file mode 100644 index 00000000..e008f1f0 --- /dev/null +++ b/apps/gallery/transfer_data.py @@ -0,0 +1,21 @@ +from django.db.models import Value, IntegerField, F +from pprint import pprint +from gallery.models import Image +from transfer.models import MercuryImages +from transfer.serializers.gallery import ImageSerializer + + +def transfer_gallery(): + queryset = MercuryImages.objects.all() + + serialized_data = ImageSerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "gallery": [transfer_gallery] +} + diff --git a/apps/location/migrations/0013_country_old_id.py b/apps/location/migrations/0013_country_old_id.py new file mode 100644 index 00000000..7a6a3029 --- /dev/null +++ b/apps/location/migrations/0013_country_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-27 06:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0012_data_migrate'), + ] + + operations = [ + migrations.AddField( + model_name='country', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0014_remove_country_old_id.py b/apps/location/migrations/0014_remove_country_old_id.py new file mode 100644 index 00000000..6b04c9b4 --- /dev/null +++ b/apps/location/migrations/0014_remove_country_old_id.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.4 on 2019-10-27 06:04 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0013_country_old_id'), + ] + + operations = [ + migrations.RemoveField( + model_name='country', + name='old_id', + ), + ] diff --git a/apps/location/migrations/0015_country_old_id.py b/apps/location/migrations/0015_country_old_id.py new file mode 100644 index 00000000..56e9b251 --- /dev/null +++ b/apps/location/migrations/0015_country_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-27 06:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0014_remove_country_old_id'), + ] + + operations = [ + migrations.AddField( + model_name='country', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0016_region_old_id.py b/apps/location/migrations/0016_region_old_id.py new file mode 100644 index 00000000..c4bd0d34 --- /dev/null +++ b/apps/location/migrations/0016_region_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-27 10:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0015_country_old_id'), + ] + + operations = [ + migrations.AddField( + model_name='region', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0017_city_old_id.py b/apps/location/migrations/0017_city_old_id.py new file mode 100644 index 00000000..b38a30bd --- /dev/null +++ b/apps/location/migrations/0017_city_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-28 04:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0016_region_old_id'), + ] + + operations = [ + migrations.AddField( + model_name='city', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0018_address_old_id.py b/apps/location/migrations/0018_address_old_id.py new file mode 100644 index 00000000..549375f4 --- /dev/null +++ b/apps/location/migrations/0018_address_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-28 05:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0017_city_old_id'), + ] + + operations = [ + migrations.AddField( + model_name='address', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0019_merge_20191030_0858.py b/apps/location/migrations/0019_merge_20191030_0858.py new file mode 100644 index 00000000..268dc08f --- /dev/null +++ b/apps/location/migrations/0019_merge_20191030_0858.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 08:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0018_address_old_id'), + ('location', '0013_wineappellation_wineregion'), + ] + + operations = [ + ] diff --git a/apps/location/migrations/0019_merge_20191030_1618.py b/apps/location/migrations/0019_merge_20191030_1618.py new file mode 100644 index 00000000..74d64c63 --- /dev/null +++ b/apps/location/migrations/0019_merge_20191030_1618.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 16:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0018_address_old_id'), + ('location', '0013_wineappellation_wineregion'), + ] + + operations = [ + ] diff --git a/apps/location/migrations/0020_merge_20191030_1714.py b/apps/location/migrations/0020_merge_20191030_1714.py new file mode 100644 index 00000000..f0b28e5a --- /dev/null +++ b/apps/location/migrations/0020_merge_20191030_1714.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 17:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0019_merge_20191030_0858'), + ('location', '0019_merge_20191030_1618'), + ] + + operations = [ + ] diff --git a/apps/location/models.py b/apps/location/models.py index ba171c27..fc81ec37 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -21,6 +21,7 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin): low_price = models.IntegerField(default=25, verbose_name=_('Low price')) high_price = models.IntegerField(default=50, verbose_name=_('High price')) languages = models.ManyToManyField(Language, verbose_name=_('Languages')) + old_id = models.IntegerField(null=True, blank=True, default=None) @property def country_id(self): @@ -43,6 +44,7 @@ class Region(models.Model): blank=True, default=None, on_delete=models.CASCADE) country = models.ForeignKey( Country, verbose_name=_('country'), on_delete=models.CASCADE) + old_id = models.IntegerField(null=True, blank=True, default=None) class Meta: """Meta class.""" @@ -76,7 +78,7 @@ class City(models.Model): _('postal code'), max_length=10, default='', help_text=_('Ex.: 350018')) is_island = models.BooleanField(_('is island'), default=False) - + old_id = models.IntegerField(null=True, blank=True, default=None) objects = CityQuerySet.as_manager() class Meta: @@ -100,6 +102,7 @@ class Address(models.Model): default='', help_text=_('Ex.: 350018')) coordinates = models.PointField( _('Coordinates'), blank=True, null=True, default=None) + old_id = models.IntegerField(null=True, blank=True, default=None) class Meta: """Meta class.""" diff --git a/apps/location/transfer.py b/apps/location/transfer.py new file mode 100644 index 00000000..fe4e8084 --- /dev/null +++ b/apps/location/transfer.py @@ -0,0 +1,141 @@ +""" + +Структура записи в card: +Название таблицы в postgresql: { + "data_type": "тип данных в таблице (словарь, объект, дочерний объект и так далее)", + "dependencies": кортеж с зависимостями от других таблиц в postgresql, + "fields": список полей для таблицы postgresql, пример: + { + "название legacy таблицы": { + список полей для переноса, пример структуры описан далее + }, + "relations": список зависимостей legacy-таблицы, пример: + { + # имеет внешний ключ на "название legacy таблицы" из "fields" + "название legacy таблицы": { + "key": ключ для связи. Строка, если тип поля в legacy таблице - ForeignKey, или кортеж из названия поля + в дочерней таблице и названия поля в родительской таблице в ином случае + "fields": { + список полей для переноса, пример структуры описан далее + } + } + } + }, + "relations": список внешних ключей таблицы postgresql, пример структуры описан далее + { + "Cities": [( + (None, "region_code"), + ("Region", "region", "code", "CharField")), + ((None, "country_code_2"), + ("Country", "country", "code", "CharField")) + ] + } +}, + + +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy) +""" + +card = { + "Country": { + "data_type": "dictionaries", + "dependencies": None, + "fields": { + "Cities": { + "code": "country_code_2", + } + } + }, + + "Region": { + # subregion_code -> code + # region_code -> parent_region + # MySQL: select distinct(country_code_2) from cities where subregion_code is not NULL; + # только у пяти стран есть понятие "subregion_code"(не NULL): fr, be, ma, aa, gr (country_code_2) + # Возможно получиться обойтись без изменений модели Region + "data_type": "dictionaries", + "dependencies": ("Country", "Region"), + "fields": { + # нету аналога для поля name + "Cities": { + "code": "region_code", + }, + }, + "relations": { + "Cities": [( + (None, "country_code_2"), + ("Country", "country", "code", "django.db.models.CharField")), + # ((None, "subregion_code"), #TODO: как сопоставлять parent_region из postgres с subregion_code из legacy ? + # ("Region", "parent_region", "code", "CharField")) + + ] + } + }, + + "City": { + "data_type": "dictionaries", + "dependencies": ("Country", "Region"), + "fields": { + "Cities": { + "name": "name", + "code": "country_code_2", + "postal_code": "zip_code", + "is_island": ("is_island", "is_island", "django.db.models.Boolean") + } + }, + "relations": { + "Cities": [( + (None, "region_code"), + ("Region", "region", "code", "django.db.models.CharField")), + ((None, "country_code_2"), + ("Country", "country", "code", "django.db.models.CharField")) + ] + } + }, + + + "Address": { + "data_type": "dictionaries", + "dependencies": ("City",), + "fields": { + # нету аналога для поля number + "Locations": { + "postal_code": "zip_code", + "coordinates": (("lat", "latitude"), ("long", "longitude")) + }, + }, + "relations": { + "Cities": [ #TODO: Locations ссылается внешним ключом на Cities + (("city", None), + ("City", "city", None, None)) + ] + } + } +} + + +used_apps = None diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py new file mode 100644 index 00000000..b19eb42b --- /dev/null +++ b/apps/location/transfer_data.py @@ -0,0 +1,123 @@ +from django.db.models import Q, QuerySet + +from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer, AddressSerializer +from transfer.models import Cities, Locations +from pprint import pprint + + +def transfer_countries(): + queryset = Cities.objects.raw("""SELECT cities.id, cities.country_code_2 + FROM cities WHERE + country_code_2 IS NOT NULL AND + country_code_2 != "" + GROUP BY cities.country_code_2""") + + queryset = [vars(query) for query in queryset] + + serialized_data = CountrySerializer(data=queryset, many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"Country serializer errors: {serialized_data.errors}") + + +def transfer_regions(): + regions_without_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, + cities.country_code_2, cities.subregion_code + FROM cities WHERE + (subregion_code IS NULL OR + subregion_code = "") AND + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "" + GROUP BY region_code""") + + regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset] + + serialized_without_subregion = RegionSerializer(data=regions_without_subregion_queryset, many=True) + if serialized_without_subregion.is_valid(): + serialized_without_subregion.save() + else: + pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") + + regions_with_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, + cities.country_code_2, cities.subregion_code + FROM cities WHERE + subregion_code IS NOT NULL AND + subregion_code != "" AND + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "" + AND cities.subregion_code in ( + SELECT region_code FROM cities WHERE + (subregion_code IS NULL OR + subregion_code = "") AND + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "" + ) + GROUP BY region_code""") + + regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset] + + serialized_with_subregion = RegionSerializer(data=regions_with_subregion_queryset, many=True) + if serialized_with_subregion.is_valid(): + serialized_with_subregion.save() + else: + pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") + + +def transfer_cities(): + queryset = Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code, + cities.is_island, cities.region_code, cities.subregion_code + FROM cities WHERE + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "" + """) + + queryset = [vars(query) for query in queryset] + + serialized_data = CitySerializer(data=queryset, many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"City serializer errors: {serialized_data.errors}") + + +def transfer_addresses(): + queryset = Locations.objects.raw("""SELECT locations.id, locations.zip_code, locations.longitude, + locations.latitude, locations.address, locations.city_id + FROM locations WHERE + locations.address != "" AND + locations.address IS NOT NULL AND + locations.city_id IS NOT NULL AND + locations.city_id IN (SELECT cities.id + FROM cities WHERE + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "")""") + + queryset = [vars(query) for query in queryset] + + serialized_data = AddressSerializer(data=queryset, many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"Address serializer errors: {serialized_data.errors}") + + + +data_types = { + "dictionaries": [ + transfer_countries, + transfer_regions, + transfer_cities, + transfer_addresses + ] +} diff --git a/apps/main/migrations/0020_merge_20191025_0423.py b/apps/main/migrations/0020_merge_20191025_0423.py new file mode 100644 index 00000000..122549cf --- /dev/null +++ b/apps/main/migrations/0020_merge_20191025_0423.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-25 04:23 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0019_auto_20191022_1359'), + ('main', '0019_award_image_url'), + ] + + operations = [ + ] diff --git a/apps/main/migrations/0023_merge_20191028_0725.py b/apps/main/migrations/0023_merge_20191028_0725.py new file mode 100644 index 00000000..e551fe2e --- /dev/null +++ b/apps/main/migrations/0023_merge_20191028_0725.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-28 07:25 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0020_merge_20191025_0423'), + ('main', '0022_auto_20191023_1113'), + ] + + operations = [ + ] diff --git a/apps/main/transfer.py b/apps/main/transfer.py new file mode 100644 index 00000000..5d9caafe --- /dev/null +++ b/apps/main/transfer.py @@ -0,0 +1,146 @@ +""" + +Структура записи в card: +Название таблицы в postgresql: { + "data_type": "тип данных в таблице (словарь, объект, дочерний объект и так далее)", + "dependencies": кортеж с зависимостями от других таблиц в postgresql, + "fields": список полей для таблицы postgresql, пример: + { + "название legacy таблицы": { + список полей для переноса, пример структуры описан далее + }, + "relations": список зависимостей legacy-таблицы, пример: + { + # имеет внешний ключ на "название legacy таблицы" из "fields" + "название legacy таблицы": { + "key": ключ для связи. Строка, если тип поля в legacy таблице - ForeignKey, или кортеж из названия поля + в дочерней таблице и названия поля в родительской таблице в ином случае + "fields": { + список полей для переноса, пример структуры описан далее + } + } + } + }, + "relations": список внешних ключей таблицы postgresql, пример структуры описан далее + { + "Cities": [( + (None, "region_code"), + ("Region", "region", "code", "CharField")), + ((None, "country_code_2"), + ("Country", "country", "code", "CharField")) + ] + } +}, + + +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +""" + +card = { + "SiteSettings": { + "data_type": "objects", + "dependencies": ("Country",), + "fields": { + "Sites": { + "subdomain": "country_code_2", + "pinterest_page_url": "pinterest_page_url", + "twitter_page_url": "twitter_page_url", + "facebook_page_url": "facebook_page_url", + "instagram_page_url": "instagram_page_url", + "contact_email": "contact_email", + "config": ("config", "django.db.models.JSONField") #TODO в качесте ключа использовать country_code_2 из legacy - ? + }, + # "relations": { + # # Как работать c отношение OneToOneField + # } + } + + }, + "Feature": { + "data_type": "objects", + "dependencies": ("SiteSettings",), + "fields": { + "Features": { + "slug": "slug" + } + }, + # поле "site_settings" ManyToManyField имеет through='SiteFeature' в postgres + # "relations": { # как работать с ManyToManyField + # + # } + }, + "SiteFeature": { + "data_type": "objects", + "dependencies": ("SiteSettings", "Feature"), + "fields": { + "SiteFeatures": { + "published": ("state", "django.db.models.BooleanField") + } + }, + "relations": { + "Sites": [( + ("site", None), + ("SiteSettings", "site_settings", None, None)) + ], + "Features": [( + ("feature", None), + ("Feature", "feature", None, None)) + ] + } + }, + "AwardType": { + "data_type": "objects", + "dependencies": ("Country",), + "fields": { + "AwardTypes": { + "name": "title" + } + }, + # вопрос с ForeignKey на Country + # "relations": { + # + # } + }, + "Award": { + "data_type": "objects", + "dependencies": ("AwardType", "ContentType"), + "fields": { + "Awards": { + "title": ("title", "django.db.models.JSONField"), + "vintage_year": "year", + } + }, + #TODO вопрос с content_type + "relations": { + "AwardTypes": [( + ("award_type", None), + ("AwardType", "award_type", None, None)) + ] + } + } +} + +used_apps = ("locations", ) \ No newline at end of file diff --git a/apps/news/admin.py b/apps/news/admin.py index 5c1cbba7..cc48a887 100644 --- a/apps/news/admin.py +++ b/apps/news/admin.py @@ -3,6 +3,7 @@ from django.conf import settings from news import models from .tasks import send_email_with_news +from utils.admin import BaseModelAdminMixin @admin.register(models.NewsType) @@ -24,9 +25,11 @@ send_email_action.short_description = "Send the selected news by email" @admin.register(models.News) -class NewsAdmin(admin.ModelAdmin): +class NewsAdmin(BaseModelAdminMixin, admin.ModelAdmin): """News admin.""" + raw_id_fields = ('address',) actions = [send_email_action] + raw_id_fields = ('news_type', 'address', 'country') @admin.register(models.NewsGallery) diff --git a/apps/news/management/__init__.py b/apps/news/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/news/management/commands/__init__.py b/apps/news/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/news/management/commands/rm_all_news.py b/apps/news/management/commands/rm_all_news.py new file mode 100644 index 00000000..6b27199e --- /dev/null +++ b/apps/news/management/commands/rm_all_news.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand + +from news.models import News + + +class Command(BaseCommand): + help = 'Remove all news from new bd' + + def handle(self, *args, **kwargs): + old_news = News.objects.all() + count = old_news.count() + old_news.delete() + self.stdout.write(self.style.WARNING(f'Deleted {count} objects.')) diff --git a/apps/news/management/commands/rm_old_news.py b/apps/news/management/commands/rm_old_news.py new file mode 100644 index 00000000..cbcf0f6a --- /dev/null +++ b/apps/news/management/commands/rm_old_news.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand + +from news.models import News + + +class Command(BaseCommand): + help = 'Remove old news from new bd' + + def handle(self, *args, **kwargs): + old_news = News.objects.exclude(old_id__isnull=True) + count = old_news.count() + old_news.delete() + self.stdout.write(self.style.WARNING(f'Deleted {count} objects.')) diff --git a/apps/news/migrations/0021_auto_20191021_1120.py b/apps/news/migrations/0021_auto_20191021_1120.py new file mode 100644 index 00000000..4aa1104e --- /dev/null +++ b/apps/news/migrations/0021_auto_20191021_1120.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-21 11:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0020_remove_news_author'), + ] + + operations = [ + migrations.AlterField( + model_name='news', + name='slug', + field=models.SlugField(max_length=255, unique=True, verbose_name='News slug'), + ), + ] diff --git a/apps/news/migrations/0023_merge_20191025_0423.py b/apps/news/migrations/0023_merge_20191025_0423.py new file mode 100644 index 00000000..66e7cd92 --- /dev/null +++ b/apps/news/migrations/0023_merge_20191025_0423.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-25 04:23 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0021_auto_20191021_1120'), + ('news', '0022_auto_20191021_1306'), + ] + + operations = [ + ] diff --git a/apps/news/migrations/0029_merge_20191025_0906.py b/apps/news/migrations/0029_merge_20191025_0906.py new file mode 100644 index 00000000..e5967a8d --- /dev/null +++ b/apps/news/migrations/0029_merge_20191025_0906.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-25 09:06 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0028_auto_20191024_1649'), + ('news', '0021_auto_20191021_1120'), + ] + + operations = [ + ] diff --git a/apps/news/migrations/0030_merge_20191028_0725.py b/apps/news/migrations/0030_merge_20191028_0725.py new file mode 100644 index 00000000..671e06d7 --- /dev/null +++ b/apps/news/migrations/0030_merge_20191028_0725.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-28 07:25 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0029_merge_20191025_0906'), + ('news', '0023_merge_20191025_0423'), + ] + + operations = [ + ] diff --git a/apps/news/migrations/0030_news_old_id.py b/apps/news/migrations/0030_news_old_id.py new file mode 100644 index 00000000..038e5e35 --- /dev/null +++ b/apps/news/migrations/0030_news_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-29 08:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0029_merge_20191025_0906'), + ] + + operations = [ + migrations.AddField( + model_name='news', + name='old_id', + field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='odl id'), + ), + ] diff --git a/apps/news/migrations/0031_merge_20191029_0858.py b/apps/news/migrations/0031_merge_20191029_0858.py new file mode 100644 index 00000000..6f00fcf0 --- /dev/null +++ b/apps/news/migrations/0031_merge_20191029_0858.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-29 08:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0030_news_old_id'), + ('news', '0030_merge_20191028_0725'), + ] + + operations = [ + ] diff --git a/apps/news/migrations/0032_auto_20191030_1149.py b/apps/news/migrations/0032_auto_20191030_1149.py new file mode 100644 index 00000000..75408f46 --- /dev/null +++ b/apps/news/migrations/0032_auto_20191030_1149.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-30 11:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0031_merge_20191029_0858'), + ] + + operations = [ + migrations.AlterField( + model_name='news', + name='old_id', + field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id'), + ), + ] diff --git a/apps/news/migrations/0032_merge_20191030_0858.py b/apps/news/migrations/0032_merge_20191030_0858.py new file mode 100644 index 00000000..2936caef --- /dev/null +++ b/apps/news/migrations/0032_merge_20191030_0858.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 08:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0029_auto_20191025_1241'), + ('news', '0031_merge_20191029_0858'), + ] + + operations = [ + ] diff --git a/apps/news/migrations/0033_merge_20191030_1618.py b/apps/news/migrations/0033_merge_20191030_1618.py new file mode 100644 index 00000000..05811754 --- /dev/null +++ b/apps/news/migrations/0033_merge_20191030_1618.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 16:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0032_auto_20191030_1149'), + ('news', '0029_auto_20191025_1241'), + ] + + operations = [ + ] diff --git a/apps/news/migrations/0034_merge_20191030_1714.py b/apps/news/migrations/0034_merge_20191030_1714.py new file mode 100644 index 00000000..5ac187ab --- /dev/null +++ b/apps/news/migrations/0034_merge_20191030_1714.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 17:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0033_merge_20191030_1618'), + ('news', '0032_merge_20191030_0858'), + ] + + operations = [ + ] diff --git a/apps/news/models.py b/apps/news/models.py index 9b7190e4..ca0f2887 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -1,11 +1,12 @@ """News app models.""" -from django.db import models from django.contrib.contenttypes import fields as generic +from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ from rest_framework.reverse import reverse -from utils.models import BaseAttributes, TJSONField, TranslatedFieldsMixin, ProjectBaseMixin + from rating.models import Rating +from utils.models import BaseAttributes, TJSONField, TranslatedFieldsMixin, ProjectBaseMixin class NewsType(models.Model): @@ -125,6 +126,7 @@ class News(BaseAttributes, TranslatedFieldsMixin): (PUBLISHED_EXCLUSIVE, _('Published exclusive')), ) + old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None) news_type = models.ForeignKey(NewsType, on_delete=models.PROTECT, verbose_name=_('news type')) title = TJSONField(blank=True, null=True, default=None, @@ -217,7 +219,6 @@ class NewsGalleryQuerySet(models.QuerySet): class NewsGallery(models.Model): - news = models.ForeignKey(News, null=True, related_name='news_gallery', on_delete=models.CASCADE, diff --git a/apps/news/transfer.py b/apps/news/transfer.py new file mode 100644 index 00000000..6a904ad9 --- /dev/null +++ b/apps/news/transfer.py @@ -0,0 +1,92 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy) +""" + +# "from django.db import models" + +# "models.PositiveIntegerField" + +# "django.db.models." + +card = { + # нету аналога для NewsType + "NewsType": { + "data_type": "news", + "dependencies": None, + "fields": { + "Pages": { + # будет только один тип новости "News" + # значения для поля "name" берутся из поля "type" legacy модели "Pages", притом type="News" + # Mysql - select distinct(type) from pages; + "name": "type" + } + } + }, + + "News": { + "data_type": "news", + # "dependencies": ("NewsType", "MetaDataContent", "Country", "Address"), + "dependencies": ("NewsType", ), + "fields": { + # нету аналогов для start, end, playlist + "Pages": { + "state": ("state", "django.db.models.PositiveSmallIntegerField"), + "template": ("template", "django.db.models.PositiveIntegerField"), + "image_url": ("attachment_file_name", "django.db.models.URLField"), + "preview_image_url": ("attachment_file_name", "django.db.models.URLField"), + + # в NewsOlds нету аналога для поля subtitle модели News, также нет аналогов для полей start, end, playlist + # "subtitle": "" + + + # Поле "description" модели News имеет тип JSONField(где ключ - это язык, а значение - новость + # на языке который указан ключом), а поле "body" NewsOlds имеет тип html-разметки + # с вставками шаблонизатора Ruby + # "description" : "body" + }, + "relations": { + "PageTexts": { + "key": "page", + "fields": { + "title": ("title", "django.db.models.JSONField"), + "description": ("body", "django.db.models.JSONField"), + "slug": ("slug", "django.db.models.SlugField") + } + } + }, + }, + # нету аналога для поля tags + "relations": { + "Pages": [ + ((None, "type"), + ("NewsType", "news_type", "name", "django.db.models.CharField")) + ] + } + }, +} + +used_apps = ("location", "main", ) diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py new file mode 100644 index 00000000..bdd1c513 --- /dev/null +++ b/apps/news/transfer_data.py @@ -0,0 +1,52 @@ +from pprint import pprint + +from django.db.models import Aggregate, CharField, Value +from django.db.models import IntegerField, F + +from news.models import NewsType +from tag.models import TagCategory +from transfer.models import PageTexts +from transfer.serializers.news import NewsSerializer + + +class GroupConcat(Aggregate): + function = 'GROUP_CONCAT' + template = '%(function)s(%(expressions)s)' + + def __init__(self, expression, **extra): + output_field = extra.pop('output_field', CharField()) + super().__init__(expression, output_field=output_field, **extra) + + def as_postgresql(self, compiler, connection): + self.function = 'STRING_AGG' + return super().as_sql(compiler, connection) + + +def transfer_news(): + news_type, _ = NewsType.objects.get_or_create(name='News') + tag_cat, _ = TagCategory.objects.get_or_create(index_name='tag', public=True) + news_type.tag_categories.add(tag_cat) + news_type.save() + + queryset = PageTexts.objects.filter( + page__type='News', + ).annotate( + tag_cat_id=Value(tag_cat.id, output_field=IntegerField()), + news_type_id=Value(news_type.id, output_field=IntegerField()), + country_code=F('page__site__country_code_2'), + news_title=F('page__root_title'), + image=F('page__attachment_suffix_url'), + template=F('page__template'), + tags=GroupConcat('page__tags__id'), + ) + + serialized_data = NewsSerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f'News serializer errors: {serialized_data.errors}') + + +data_types = { + 'news': [transfer_news] +} diff --git a/apps/notification/transfer.py b/apps/notification/transfer.py new file mode 100644 index 00000000..9fa47a20 --- /dev/null +++ b/apps/notification/transfer.py @@ -0,0 +1,48 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy) +""" + +card = { + # нету аналога для NewsType + "Subscriber": { + "data_type": "objects", + # "dependencies": ("User", ), + "dependencies": None, + "fields": { + "EmailAddresses": { + "email": "email", + "state": ("state", "django.db.models.PositiveIntegerField") # из legacy брать только те записи у которых state=usable + }, + # "relations": [ + # # отложено до выяснения Уточнения и вопросы по мигратору(Как поступать со сбором данных) + # # "user": "" + # ] + } + }, +} + +used_apps = ("account", ) diff --git a/apps/notification/transfer_data.py b/apps/notification/transfer_data.py new file mode 100644 index 00000000..3dd69f56 --- /dev/null +++ b/apps/notification/transfer_data.py @@ -0,0 +1,21 @@ +from transfer.serializers.notification import SubscriberSerializer +from notification.models import Subscriber +from transfer.models import EmailAddresses +from django.db.models import Value, IntegerField, F +from pprint import pprint + + +def transfer_subscriber(): + queryset = EmailAddresses.objects.filter(state="usable") + + serialized_data = SubscriberSerializer(data=list(queryset.values()), many=True) + + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "subscriber": [transfer_subscriber] +} diff --git a/apps/partner/transfer.py b/apps/partner/transfer.py new file mode 100644 index 00000000..f89c7970 --- /dev/null +++ b/apps/partner/transfer.py @@ -0,0 +1,41 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +""" + +card = { + "Partner": { + "data_type": "objects", + "dependencies": None, + "fields": { + # из EstablishmentBacklinks выбирать только те у которых type="Partner" + "EstablishmentBacklinks": { + "url": "backlink_url", + }, + } + }, +} + +used_apps = None diff --git a/apps/partner/transfer_data.py b/apps/partner/transfer_data.py new file mode 100644 index 00000000..084fd7b1 --- /dev/null +++ b/apps/partner/transfer_data.py @@ -0,0 +1,19 @@ +from django.db.models import Value, IntegerField, F +from pprint import pprint +from transfer.models import EstablishmentBacklinks +from transfer.serializers.partner import PartnerSerializer + + +def transfer_partner(): + queryset = EstablishmentBacklinks.objects.filter(type="Partner") + + serialized_data = PartnerSerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "partner": [transfer_partner] +} diff --git a/apps/product/admin.py b/apps/product/admin.py index b3dbc0cd..b77f2cbc 100644 --- a/apps/product/admin.py +++ b/apps/product/admin.py @@ -1,10 +1,11 @@ """Product admin conf.""" from django.contrib import admin +from utils.admin import BaseModelAdminMixin from .models import Product, ProductType, ProductSubType @admin.register(Product) -class ProductAdmin(admin.ModelAdmin): +class ProductAdmin(BaseModelAdminMixin, admin.ModelAdmin): """Admin page for model Product.""" diff --git a/apps/product/filters.py b/apps/product/filters.py index 7894a6e6..a30147eb 100644 --- a/apps/product/filters.py +++ b/apps/product/filters.py @@ -5,7 +5,7 @@ from django_filters import rest_framework as filters from product import models -class ProductListFilterSet(filters.FilterSet): +class ProductFilterSet(filters.FilterSet): """Product filter set.""" establishment_id = filters.NumberFilter() diff --git a/apps/product/management/__init__.py b/apps/product/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/product/management/commands/__init__.py b/apps/product/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/product/management/commands/add_product.py b/apps/product/management/commands/add_product.py new file mode 100644 index 00000000..981487ec --- /dev/null +++ b/apps/product/management/commands/add_product.py @@ -0,0 +1,54 @@ +from django.core.management.base import BaseCommand +from product.models import ProductType, ProductSubType + + +def add_type(): + product_type = ProductType.objects.create( + name={'"en-GB"': "Wine"}, + index_name=ProductType.WINE + ) + return product_type.save() + + +def add_subtype(id_type): + subtypes = ProductSubType.objects.bulk_create([ + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.EXTRA_BRUT}, + index_name=ProductSubType.EXTRA_BRUT), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.BRUT}, + index_name=ProductSubType.BRUT), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.BRUT_NATURE}, + index_name=ProductSubType.BRUT_NATURE), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.DEMI_SEC}, + index_name=ProductSubType.DEMI_SEC), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.EXTRA_DRY}, + index_name=ProductSubType.EXTRA_DRY), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.DOSAGE_ZERO}, + index_name=ProductSubType.DOSAGE_ZERO), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.SEC}, + index_name=ProductSubType.SEC), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.SEC}, + index_name=ProductSubType.SEC), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.MOELLEUX}, + index_name=ProductSubType.MOELLEUX), + ]) + + +class Command(BaseCommand): + help = 'Add product data' + + def handle(self, *args, **kwarg): + product_type = add_type() + add_subtype(product_type.id) + + + + diff --git a/apps/product/migrations/0002_product_slug.py b/apps/product/migrations/0002_product_slug.py new file mode 100644 index 00000000..a4605464 --- /dev/null +++ b/apps/product/migrations/0002_product_slug.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-29 14:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('product', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='product', + name='slug', + field=models.SlugField(max_length=255, null=True, unique=True, verbose_name='Establishment slug'), + ), + ] diff --git a/apps/product/models.py b/apps/product/models.py index 40007f18..332a92f6 100644 --- a/apps/product/models.py +++ b/apps/product/models.py @@ -1,5 +1,6 @@ """Product app models.""" from django.db import models +from django.contrib.contenttypes import fields as generic from django.core.exceptions import ValidationError from django.contrib.postgres.fields import JSONField from django.utils.translation import gettext_lazy as _ @@ -45,10 +46,28 @@ class ProductSubType(TranslatedFieldsMixin, ProjectBaseMixin): # INDEX NAME CHOICES RUM = 'rum' OTHER = 'other' + EXTRA_BRUT = 'extra brut' + BRUT = 'brut' + BRUT_NATURE = 'brut nature' + DEMI_SEC = 'demi-sec' + EXTRA_DRY = 'Extra Dry' + DOSAGE_ZERO = 'dosage zero' + SEC = 'sec' + DOUX = 'doux' + MOELLEUX= 'moelleux' INDEX_NAME_TYPES = ( (RUM, _('Rum')), (OTHER, _('Other')), + (EXTRA_BRUT, _('extra brut')), + (BRUT, _('brut')), + (BRUT_NATURE, _('brut nature')), + (DEMI_SEC, _('demi-sec')), + (EXTRA_DRY, _('Extra Dry')), + (DOSAGE_ZERO, _('dosage zero')), + (SEC, _('sec')), + (DOUX, _('doux')), + (MOELLEUX, _('moelleux')) ) product_type = models.ForeignKey(ProductType, on_delete=models.CASCADE, @@ -141,6 +160,9 @@ class Product(TranslatedFieldsMixin, BaseAttributes): wine_appellation = models.ForeignKey('location.WineAppellation', on_delete=models.PROTECT, blank=True, null=True, verbose_name=_('wine appellation')) + slug = models.SlugField(unique=True, max_length=255, null=True, + verbose_name=_('Establishment slug')) + favorites = generic.GenericRelation(to='favorites.Favorites') objects = ProductManager.from_queryset(ProductQuerySet)() diff --git a/apps/product/serializers/common.py b/apps/product/serializers/common.py index fb2f7aeb..7ebfaa28 100644 --- a/apps/product/serializers/common.py +++ b/apps/product/serializers/common.py @@ -1,9 +1,11 @@ """Product app serializers.""" from rest_framework import serializers -from utils.serializers import TranslatedField +from utils.serializers import TranslatedField, FavoritesCreateSerializer from product.models import Product, ProductSubType, ProductType +from utils import exceptions as utils_exceptions +from django.utils.translation import gettext_lazy as _ from location.serializers import (WineRegionBaseSerializer, WineAppellationBaseSerializer, - CountrySimpleSerializer) + CountrySimpleSerializer) class ProductSubTypeBaseSerializer(serializers.ModelSerializer): @@ -50,6 +52,7 @@ class ProductBaseSerializer(serializers.ModelSerializer): model = Product fields = [ 'id', + 'slug', 'name_translated', 'category_display', 'description_translated', @@ -61,3 +64,33 @@ class ProductBaseSerializer(serializers.ModelSerializer): 'wine_appellation', 'available_countries', ] + + +class ProductFavoritesCreateSerializer(FavoritesCreateSerializer): + """Serializer to create favorite object w/ model Product.""" + + def validate(self, attrs): + """Overridden validate method""" + # Check establishment object + product_qs = Product.objects.filter(slug=self.slug) + + # Check establishment obj by slug from lookup_kwarg + if not product_qs.exists(): + raise serializers.ValidationError({'detail': _('Object not found.')}) + else: + product = product_qs.first() + + # Check existence in favorites + if product.favorites.filter(user=self.user).exists(): + raise utils_exceptions.FavoritesError() + + attrs['product'] = product + return attrs + + def create(self, validated_data, *args, **kwargs): + """Overridden create method""" + validated_data.update({ + 'user': self.user, + 'content_object': validated_data.pop('product') + }) + return super().create(validated_data) \ No newline at end of file diff --git a/apps/product/urls/common.py b/apps/product/urls/common.py index 57abf4f0..d0dbb8a9 100644 --- a/apps/product/urls/common.py +++ b/apps/product/urls/common.py @@ -6,5 +6,7 @@ from product import views app_name = 'product' urlpatterns = [ - path('', views.ProductListView.as_view(), name='list') + path('', views.ProductListView.as_view(), name='list'), + path('slug//favorites/', views.CreateFavoriteProductView.as_view(), + name='create-destroy-favorites') ] diff --git a/apps/product/views/common.py b/apps/product/views/common.py index 403781e4..63b9677f 100644 --- a/apps/product/views/common.py +++ b/apps/product/views/common.py @@ -1,5 +1,6 @@ """Product app views.""" from rest_framework import generics, permissions +from django.shortcuts import get_object_or_404 from product.models import Product from product import serializers from product import filters @@ -17,4 +18,22 @@ class ProductListView(ProductBaseView, generics.ListAPIView): """List view for model Product.""" permission_classes = (permissions.AllowAny, ) serializer_class = serializers.ProductBaseSerializer - filter_class = filters.ProductListFilterSet + filter_class = filters.ProductFilterSet + + +class CreateFavoriteProductView(generics.CreateAPIView, + generics.DestroyAPIView): + """View for create/destroy product in favorites.""" + serializer_class = serializers.ProductFavoritesCreateSerializer + lookup_field = 'slug' + + def get_object(self): + """ + Returns the object the view is displaying. + """ + product = get_object_or_404(Product, slug=self.kwargs['slug']) + favorites = get_object_or_404(product.favorites.filter(user=self.request.user)) + + # May raise a permission denied + self.check_object_permissions(self.request, favorites) + return favorites diff --git a/apps/recipe/models.py b/apps/recipe/models.py index 6df7adc2..349fed7b 100644 --- a/apps/recipe/models.py +++ b/apps/recipe/models.py @@ -15,14 +15,12 @@ class RecipeQuerySet(models.QuerySet): def annotate_in_favorites(self, user): """Annotate flag in_favorites""" - favorite_establishments = [] + favorite_recipe_ids = [] if user.is_authenticated: - favorite_establishments = user.favorites.by_content_type(app_label='recipe', - model='recipe') \ - .values_list('object_id', flat=True) + favorite_recipe_ids = user.favorite_recipe_ids return self.annotate(in_favorites=models.Case( models.When( - id__in=favorite_establishments, + id__in=favorite_recipe_ids, then=True), default=False, output_field=models.BooleanField(default=False))) diff --git a/apps/recipe/transfer.py b/apps/recipe/transfer.py new file mode 100644 index 00000000..7b0d4bf8 --- /dev/null +++ b/apps/recipe/transfer.py @@ -0,0 +1,86 @@ +""" + +Структура записи в card: +Название таблицы в postgresql: { + "data_type": "тип данных в таблице (словарь, объект, дочерний объект и так далее)", + "dependencies": кортеж с зависимостями от других таблиц в postgresql, + "fields": список полей для таблицы postgresql, пример: + { + "название legacy таблицы": { + список полей для переноса, пример структуры описан далее + }, + "relations": список зависимостей legacy-таблицы, пример: + { + # имеет внешний ключ на "название legacy таблицы" из "fields" + "название legacy таблицы": { + "key": ключ для связи. Строка, если тип поля в legacy таблице - ForeignKey, или кортеж из названия поля + в дочерней таблице и названия поля в родительской таблице в ином случае + "fields": { + список полей для переноса, пример структуры описан далее + } + } + } + }, + "relations": список внешних ключей таблицы postgresql, пример структуры описан далее + { + "Cities": [( + (None, "region_code"), + ("Region", "region", "code", "CharField")), + ((None, "country_code_2"), + ("Country", "country", "code", "CharField")) + ] + } +}, + + +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy) +""" + +# значения берутся из legacy модели "Pages", притом type="Recipe" +card = { + "Recipe": { + "data_type": "objects", + "dependencies": None, + "fields": { + "Pages": { + "state": ("state", "django.db.models.PositiveSmallIntegerField") + }, + "relations": { + "PageTexts": { + "key": "page", + "fields": { + "title": ("title", "django.db.models.JSONField"), + "description": ("body", "django.db.models.JSONField") + } + } + } + } + } +} + + +used_apps = None diff --git a/apps/recipe/transfer_data.py b/apps/recipe/transfer_data.py new file mode 100644 index 00000000..4a2e97d6 --- /dev/null +++ b/apps/recipe/transfer_data.py @@ -0,0 +1,19 @@ +from django.db.models import Value, IntegerField, F +from pprint import pprint +from transfer.models import PageTexts +from transfer.serializers.recipe import RecipeSerializer + + +def transfer_recipe(): + queryset = PageTexts.objects.filter(page__type="Recipe") + + serialized_data = RecipeSerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "recipe": [transfer_recipe] +} diff --git a/apps/review/admin.py b/apps/review/admin.py index 5820f557..2a7326ae 100644 --- a/apps/review/admin.py +++ b/apps/review/admin.py @@ -1,8 +1,9 @@ """Admin page for app Review""" from . import models from django.contrib import admin +from utils.admin import BaseModelAdminMixin @admin.register(models.Review) -class ReviewAdminModel(admin.ModelAdmin): +class ReviewAdminModel(BaseModelAdminMixin, admin.ModelAdmin): """Admin model for model Review.""" diff --git a/apps/review/migrations/0005_review_old_id.py b/apps/review/migrations/0005_review_old_id.py new file mode 100644 index 00000000..c0e92f04 --- /dev/null +++ b/apps/review/migrations/0005_review_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-31 06:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('review', '0004_review_country'), + ] + + operations = [ + migrations.AddField( + model_name='review', + name='old_id', + field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id'), + ), + ] diff --git a/apps/review/models.py b/apps/review/models.py index 4c7f3385..194c6d58 100644 --- a/apps/review/models.py +++ b/apps/review/models.py @@ -68,6 +68,9 @@ class Review(BaseAttributes, TranslatedFieldsMixin): country = models.ForeignKey('location.Country', on_delete=models.CASCADE, related_name='country', verbose_name=_('Country'), null=True) + + old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None) + objects = ReviewQuerySet.as_manager() class Meta: diff --git a/apps/review/transfer.py b/apps/review/transfer.py new file mode 100644 index 00000000..fa56c4b2 --- /dev/null +++ b/apps/review/transfer.py @@ -0,0 +1,64 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +""" + +card = { + # как работать с GenericForeignKey(content_type) - ? + # как работать с ForeignKey на самого себя(self), поле "child" + # вопрос с внешним ключом language на таблицу Language + "Review": { + "data_type": "objects", + "dependencies": ("User", "Language", "Review"), + "fields": { + "Reviews": { + "published_at": "published_at", + "vintage": "vintage", + "status": ("aasm_state", "django.db.models.PositiveSmallIntegerField") + + # "content_object": "" + }, + "relations": { + "ReviewTexts": { + "key": "review", + "fields": { + # полу text в модели Review имеет тип TJSONField, а поле text в модели ReviewTexts имеет тип TextField + # при их сопоставлении использовать поле locale модели ReviewTexts + "text": ("text", "django.db.models.JSONField") + } + } + }, + }, + "relations": { + "Accounts": [ + (("account", None), + ("User", "reviewer", None, None)) + ] + } + } + +} + +used_apps = ("account", "translation") \ No newline at end of file diff --git a/apps/review/transfer_data.py b/apps/review/transfer_data.py new file mode 100644 index 00000000..b3a0e833 --- /dev/null +++ b/apps/review/transfer_data.py @@ -0,0 +1,101 @@ +from transfer.models import Reviews, ReviewTexts +from transfer.serializers.reviews import LanguageSerializer, ReviewSerializer, Establishment +from pprint import pprint +import json + + +def transfer_languages(): + queryset = ReviewTexts.objects.raw("""SELECT id, locale + FROM review_texts + WHERE CHAR_LENGTH(locale) = 5 + AND review_texts.locale IS NOT NULL GROUP BY locale""") + + queryset = [vars(query) for query in queryset] + + serialized_data = LanguageSerializer(data=queryset, many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"Language serializer errors: {serialized_data.errors}") + + +def transfer_reviews(): + queryset = Reviews.objects.raw("""SELECT reviews.id, reviews.vintage, reviews.establishment_id, + reviews.reviewer_id, review_texts.text AS text, reviews.mark, reviews.published_at, + review_texts.created_at AS published, review_texts.locale AS locale, + reviews.aasm_state + FROM reviews + LEFT OUTER JOIN review_texts + ON (reviews.id = review_texts.review_id) + WHERE reviews.reviewer_id > 0 + AND reviews.reviewer_id IS NOT NULL + AND review_texts.text IS NOT NULL + AND review_texts.locale IS NOT NULL + AND reviews.mark IS NOT NULL + AND reviews.reviewer_id IN ( + SELECT accounts.id + FROM accounts + WHERE accounts.confirmed_at IS NOT NULL + AND NOT accounts.email IN ( + "cyril@tomatic.net", + "cyril2@tomatic.net", + "cyril2@tomatic.net", + "d.sadykova@id-east.ru", + "d.sadykova@octopod.ru", + "n.yurchenko@id-east.ru" + )) + AND reviews.establishment_id IN ( + SELECT establishments.id + FROM establishments + INNER JOIN locations + ON (establishments.location_id = locations.id) + INNER JOIN cities + ON (locations.city_id = cities.id) + WHERE UPPER(cities.name) LIKE UPPER("%%paris%%") + AND NOT establishments.type = "Wineyard" + AND establishments.type IS NOT NULL AND locations.timezone IS NOT NULL + ) + ORDER BY review_texts.created_at DESC + """) + + queryset_result = [] + establishments_mark_list = {} + + for query in queryset: + query = vars(query) + if query['establishment_id'] not in establishments_mark_list.keys(): + if "aasm_state" in query and query['aasm_state'] is not None and query['aasm_state'] == "published": + establishments_mark_list[query['establishment_id']] = [ + int(query['mark']), + json.dumps({query['locale']: query['text']}) + ] + else: + establishments_mark_list[query['establishment_id']] = int(query['mark']) + del(query['mark']) + queryset_result.append(query) + + serialized_data = ReviewSerializer(data=queryset_result, many=True) + + if serialized_data.is_valid(): + serialized_data.save() + + for establishment_id, mark in establishments_mark_list.items(): + try: + establishment = Establishment.objects.get(old_id=establishment_id) + except Establishment.DoesNotExist: + continue + if isinstance(mark, list): + mark, review_text = mark + + establishment.public_mark = mark + establishment.save() + else: + pprint(serialized_data.errors) + + +data_types = { + "overlook": [ + transfer_languages, + transfer_reviews + ] +} diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py index 88f7e44a..e4e0c937 100644 --- a/apps/search_indexes/documents/establishment.py +++ b/apps/search_indexes/documents/establishment.py @@ -10,16 +10,6 @@ EstablishmentIndex = Index(settings.ELASTICSEARCH_INDEX_NAMES.get(__name__, EstablishmentIndex.settings(number_of_shards=1, number_of_replicas=1) -# todo: check & refactor -class ObjectField(fields.ObjectField): - - def get_value_from_instance(self, *args, **kwargs): - value = super(ObjectField, self).get_value_from_instance(*args, **kwargs) - if value == {}: - return None - return value - - @EstablishmentIndex.doc_type class EstablishmentDocument(Document): """Establishment document.""" @@ -47,7 +37,7 @@ class EstablishmentDocument(Document): works_noon = fields.ListField(fields.IntegerField( attr='works_noon' )) - works_now = fields.BooleanField(attr='works_now') + # works_now = fields.BooleanField(attr='works_now') tags = fields.ObjectField( properties={ 'id': fields.IntegerField(attr='id'), @@ -63,7 +53,7 @@ class EstablishmentDocument(Document): 'closed_at': fields.KeywordField(attr='closed_at_str'), } )) - address = ObjectField( + address = fields.ObjectField( properties={ 'id': fields.IntegerField(), 'street_name_1': fields.TextField( @@ -93,13 +83,6 @@ class EstablishmentDocument(Document): ), }, ) - # todo: need to fix - # collections = fields.ObjectField( - # properties={ - # 'id': fields.IntegerField(attr='collection.id'), - # 'collection_type': fields.IntegerField(attr='collection.collection_type'), - # }, - # multi=True) class Django: @@ -108,6 +91,7 @@ class EstablishmentDocument(Document): 'id', 'name', 'name_translated', + 'is_publish', 'price_level', 'toque_number', 'public_mark', @@ -115,4 +99,4 @@ class EstablishmentDocument(Document): ) def get_queryset(self): - return super().get_queryset().published().with_es_related() + return super().get_queryset().with_es_related() diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py index cb1fe735..a356f99b 100644 --- a/apps/search_indexes/serializers.py +++ b/apps/search_indexes/serializers.py @@ -1,5 +1,6 @@ """Search indexes serializers.""" from rest_framework import serializers +from elasticsearch_dsl import AttrDict from django_elasticsearch_dsl_drf.serializers import DocumentSerializer from news.serializers import NewsTypeSerializer from search_indexes.documents import EstablishmentDocument, NewsDocument @@ -13,6 +14,8 @@ class TagsDocumentSerializer(serializers.Serializer): label_translated = serializers.SerializerMethodField() def get_label_translated(self, obj): + if isinstance(obj, dict): + return get_translated_value(obj.get('label')) return get_translated_value(obj.label) @@ -29,6 +32,13 @@ class AddressDocumentSerializer(serializers.Serializer): geo_lon = serializers.FloatField(allow_null=True, source='coordinates.lon') geo_lat = serializers.FloatField(allow_null=True, source='coordinates.lat') + # todo: refator + def to_representation(self, instance): + if instance != AttrDict(d={}) or \ + (isinstance(instance, dict) and len(instance) != 0): + return super().to_representation(instance) + return None + class ScheduleDocumentSerializer(serializers.Serializer): """Schedule serializer for ES Document""" @@ -97,7 +107,7 @@ class EstablishmentDocumentSerializer(DocumentSerializer): 'schedule', 'works_noon', 'works_evening', - 'works_now', + # 'works_now', # 'collections', # 'establishment_type', # 'establishment_subtypes', diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py index fd1c5b7a..1cbf9d05 100644 --- a/apps/search_indexes/views.py +++ b/apps/search_indexes/views.py @@ -3,12 +3,13 @@ from rest_framework import permissions from django_elasticsearch_dsl_drf import constants from django_elasticsearch_dsl_drf.filter_backends import ( FilteringFilterBackend, - GeoSpatialFilteringFilterBackend + GeoSpatialFilteringFilterBackend, + DefaultOrderingFilterBackend, ) from django_elasticsearch_dsl_drf.viewsets import BaseDocumentViewSet from search_indexes import serializers, filters from search_indexes.documents import EstablishmentDocument, NewsDocument -from utils.pagination import ProjectPageNumberPagination +from utils.pagination import ProjectMobilePagination class NewsDocumentViewSet(BaseDocumentViewSet): @@ -16,7 +17,7 @@ class NewsDocumentViewSet(BaseDocumentViewSet): document = NewsDocument lookup_field = 'slug' - pagination_class = ProjectPageNumberPagination + pagination_class = ProjectMobilePagination permission_classes = (permissions.AllowAny,) serializer_class = serializers.NewsDocumentSerializer ordering = ('id',) @@ -53,15 +54,20 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): document = EstablishmentDocument lookup_field = 'slug' - pagination_class = ProjectPageNumberPagination + pagination_class = ProjectMobilePagination permission_classes = (permissions.AllowAny,) serializer_class = serializers.EstablishmentDocumentSerializer - ordering = ('id',) + + def get_queryset(self): + qs = super(EstablishmentDocumentViewSet, self).get_queryset() + qs = qs.filter('match', is_publish=True) + return qs filter_backends = [ FilteringFilterBackend, filters.CustomSearchFilterBackend, GeoSpatialFilteringFilterBackend, + DefaultOrderingFilterBackend, ] search_fields = { @@ -72,6 +78,7 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): translated_search_fields = ( 'description', ) + ordering = 'id' filter_fields = { 'slug': 'slug', 'tag': { @@ -145,12 +152,12 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): constants.LOOKUP_QUERY_IN, ], }, - 'works_now': { - 'field': 'works_now', - 'lookups': [ - constants.LOOKUP_FILTER_TERM, - ] - }, + # 'works_now': { + # 'field': 'works_now', + # 'lookups': [ + # constants.LOOKUP_FILTER_TERM, + # ] + # }, } geo_spatial_filter_fields = { diff --git a/apps/tag/management/__init__.py b/apps/tag/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/tag/management/commands/__init__.py b/apps/tag/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/tag/management/commands/add_tags.py b/apps/tag/management/commands/add_tags.py new file mode 100644 index 00000000..89ee89a1 --- /dev/null +++ b/apps/tag/management/commands/add_tags.py @@ -0,0 +1,69 @@ +from django.core.management.base import BaseCommand + +from establishment.models import Establishment, EstablishmentType +from transfer import models as legacy +from tag.models import Tag, TagCategory + + +class Command(BaseCommand): + help = 'Add tags values from old db to new db' + + def handle(self, *args, **kwargs): + + existing_establishment = Establishment.objects.filter(old_id__isnull=False) + ESTABLISHMENT = 1 + SHOP = 2 + RESTAURANT = 3 + WINEYARD = 4 + + MAPPER = { + RESTAURANT: EstablishmentType.RESTAURANT, + WINEYARD: EstablishmentType.PRODUCER, + } + + mapper_values_meta = legacy.KeyValueMetadatumKeyValueMetadatumEstablishments.objects.all() + for key, value in MAPPER.items(): + values_meta_id_list = mapper_values_meta.filter( + key_value_metadatum_establishment_id=key + ).values_list('key_value_metadatum_id') + + est_type, _ = EstablishmentType.objects.get_or_create(index_name=value) + + key_value_metadata = legacy.KeyValueMetadata.objects.filter( + id__in=values_meta_id_list) + + # create TagCategory + for key_value in key_value_metadata: + tag_category, created = TagCategory.objects.get_or_create( + index_name=key_value.key_name, + ) + + if created: + tag_category.label = { + 'en-GB': key_value.key_name, + 'fr-FR': key_value.key_name, + 'ru-RU': key_value.key_name, + }, + tag_category.value_type = key_value.value_type + tag_category.save() + est_type.tag_categories.add( + tag_category + ) + + # create Tag + for tag in key_value.metadata_set.filter( + establishment__id__in=list(existing_establishment.values_list('old_id', flat=True))): + + new_tag, _ = Tag.objects.get_or_create( + label={ + 'en-GB': tag.value, + 'fr-FR': tag.value, + 'ru-RU': tag.value, + }, + value=tag.value, + category=tag_category, + ) + est = existing_establishment.get(old_id=tag.establishment_id) + est.tags.add(new_tag) + est.save() + diff --git a/apps/tag/migrations/0006_auto_20191030_1151.py b/apps/tag/migrations/0006_auto_20191030_1151.py new file mode 100644 index 00000000..b11a4c20 --- /dev/null +++ b/apps/tag/migrations/0006_auto_20191030_1151.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.4 on 2019-10-30 11:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tag', '0005_tagcategory_name_indexing'), + ] + + operations = [ + migrations.AddField( + model_name='tag', + name='value', + field=models.CharField(blank=True, default=None, max_length=255, null=True, unique=True, verbose_name='indexing name'), + ), + migrations.AddField( + model_name='tagcategory', + name='value_type', + field=models.CharField(choices=[('string', 'string'), ('list', 'list'), ('integer', 'integer')], default='list', max_length=255, verbose_name='value type'), + ), + ] diff --git a/apps/tag/migrations/0007_auto_20191030_1514.py b/apps/tag/migrations/0007_auto_20191030_1514.py new file mode 100644 index 00000000..1bcb3deb --- /dev/null +++ b/apps/tag/migrations/0007_auto_20191030_1514.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-30 15:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tag', '0006_auto_20191030_1151'), + ] + + operations = [ + migrations.AlterField( + model_name='tag', + name='value', + field=models.CharField(blank=True, default=None, max_length=255, null=True, verbose_name='indexing name'), + ), + ] diff --git a/apps/tag/models.py b/apps/tag/models.py index 26079849..e3967f0f 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -19,6 +19,8 @@ class Tag(TranslatedFieldsMixin, models.Model): label = TJSONField(blank=True, null=True, default=None, verbose_name=_('label'), help_text='{"en-GB":"some text"}') + value = models.CharField(_('indexing name'), max_length=255, blank=True, + null=True, default=None) category = models.ForeignKey('TagCategory', on_delete=models.CASCADE, null=True, related_name='tags', verbose_name=_('Category')) @@ -72,6 +74,16 @@ class TagCategoryQuerySet(models.QuerySet): class TagCategory(TranslatedFieldsMixin, models.Model): """Tag base category model.""" + STRING = 'string' + LIST = 'list' + INTEGER = 'integer' + + VALUE_TYPE_CHOICES = ( + (STRING, _('string')), + (LIST, _('list')), + (INTEGER, _('integer')), + ) + label = TJSONField(blank=True, null=True, default=None, verbose_name=_('label'), help_text='{"en-GB":"some text"}') @@ -80,7 +92,10 @@ class TagCategory(TranslatedFieldsMixin, models.Model): default=None) public = models.BooleanField(default=False) index_name = models.CharField(max_length=255, blank=True, null=True, - verbose_name=_('indexing name'), unique=True) + verbose_name=_('indexing name'), unique=True) + + value_type = models.CharField(_('value type'), max_length=255, + choices=VALUE_TYPE_CHOICES, default=LIST, ) objects = TagCategoryQuerySet.as_manager() diff --git a/apps/tag/serializers.py b/apps/tag/serializers.py index 790c8926..c09f42f1 100644 --- a/apps/tag/serializers.py +++ b/apps/tag/serializers.py @@ -12,7 +12,9 @@ from utils.serializers import TranslatedField class TagBaseSerializer(serializers.ModelSerializer): """Serializer for model Tag.""" - label_translated = TranslatedField() + # todo: refactor this + # label_translated = TranslatedField() + label_translated = serializers.CharField(source='value') class Meta: """Meta class.""" @@ -39,7 +41,9 @@ class TagBackOfficeSerializer(TagBaseSerializer): class TagCategoryBaseSerializer(serializers.ModelSerializer): """Serializer for model TagCategory.""" - label_translated = TranslatedField() + # todo: refactor this + # label_translated = TranslatedField() + label_translated = serializers.CharField(source='index_name') tags = TagBaseSerializer(many=True, read_only=True) class Meta: diff --git a/apps/timetable/serialziers.py b/apps/timetable/serialziers.py index babe33c1..37725e1d 100644 --- a/apps/timetable/serialziers.py +++ b/apps/timetable/serialziers.py @@ -27,6 +27,7 @@ class ScheduleRUDSerializer(serializers.ModelSerializer): fields = [ 'id', 'weekday_display', + 'weekday', 'lunch_start', 'lunch_end', 'dinner_start', diff --git a/apps/timetable/transfer.py b/apps/timetable/transfer.py new file mode 100644 index 00000000..915b9ad5 --- /dev/null +++ b/apps/timetable/transfer.py @@ -0,0 +1,78 @@ +""" + +Структура записи в card: +Название таблицы в postgresql: { + "data_type": "тип данных в таблице (словарь, объект, дочерний объект и так далее)", + "dependencies": кортеж с зависимостями от других таблиц в postgresql, + "fields": список полей для таблицы postgresql, пример: + { + "название legacy таблицы": { + список полей для переноса, пример структуры описан далее + }, + "relations": список зависимостей legacy-таблицы, пример: + { + # имеет внешний ключ на "название legacy таблицы" из "fields" + "название legacy таблицы": { + "key": ключ для связи. Строка, если тип поля в legacy таблице - ForeignKey, или кортеж из названия поля + в дочерней таблице и названия поля в родительской таблице в ином случае + "fields": { + список полей для переноса, пример структуры описан далее + } + } + } + }, + "relations": список внешних ключей таблицы postgresql, пример структуры описан далее + { + "Cities": [( + (None, "region_code"), + ("Region", "region", "code", "CharField")), + ((None, "country_code_2"), + ("Country", "country", "code", "CharField")) + ] + } +}, + + +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +""" + +card = { + "Timetable": { + "data_type": "objects", + "dependencies": None, + "fields": { + "Schedules": { + # нет аналогов для weekday, opening_at, closed_at + "lunch_start": "lunch_start", + "lunch_end": "lunch_end", + "diner_start": "diner_start", + "diner_end": "diner_end" + } + } + } +} + +used_apps = None \ No newline at end of file diff --git a/apps/transfer/__init__.py b/apps/transfer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/transfer/admin.py b/apps/transfer/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/apps/transfer/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/apps/transfer/apps.py b/apps/transfer/apps.py new file mode 100644 index 00000000..321ffb83 --- /dev/null +++ b/apps/transfer/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig +from django.utils.translation import gettext_lazy as _ + + +class TransferConfig(AppConfig): + name = 'transfer' + verbose_name = _('Transfer') diff --git a/apps/transfer/management/__init__.py b/apps/transfer/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/transfer/management/commands/__init__.py b/apps/transfer/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py new file mode 100644 index 00000000..f3abbfab --- /dev/null +++ b/apps/transfer/management/commands/transfer.py @@ -0,0 +1,44 @@ +from django.core.management.base import BaseCommand +from transfer.utils import transfer_objects + + +class Command(BaseCommand): + help = 'Transfer data between databases' + + """Типы данных для трансфера + ВНИМАНИЕ: первые буквы типов данных должны быть уникальны! + """ + DATA_TYPES = [ + 'dictionaries', + 'news', + 'account', + 'subscriber', + 'recipe', + 'partner', + 'establishment', + 'gallery', + 'commercial', + 'overlook', + 'tmp', + 'menu', + 'location_establishment' + ] + + def handle(self, *args, **options): + """Находим включённую опцию путём пересечения множества типов данных и множества включённых опций""" + data_type = list(set(option for option in options.keys() if options[option]) & set(self.DATA_TYPES)) + if len(data_type) != 1: + print("You must set correct option!\r\nYou can get options list with \r\n\r\n\tmanage.py help transfer\r\n") + exit(1) + transfer_objects(data_type[0]) + + def add_arguments(self, parser): + """Добавляем опции к команде, основываясь на типах данных, определённых в DATA_TYPES""" + for option_type in self.DATA_TYPES: + parser.add_argument( + f'-{option_type[:1]}', + f'--{option_type}', + action='store_true', + default=False, + help=f'Transfer {option_type} objects' + ) diff --git a/apps/transfer/mixins.py b/apps/transfer/mixins.py new file mode 100644 index 00000000..a8e8287a --- /dev/null +++ b/apps/transfer/mixins.py @@ -0,0 +1,26 @@ +from django.db import models +from django.forms.models import model_to_dict + + +class SecondDbManager(models.Manager): + def get_queryset(self): + qs = super().get_queryset() + + # if `use_db` is set on model use that for choosing the DB + if hasattr(self.model, 'use_db'): + qs = qs.using(self.model.use_db) + + return qs + + +class MigrateMixin(models.Model): + """Mixin to data transfer from legacy database""" + use_db = 'legacy' + objects = SecondDbManager() + + def _parse_instance_fields(self, fields): + model_dict = model_to_dict(self, fields) + print(getattr(self, "using", "Using not found")) + + class Meta: + abstract = True diff --git a/apps/transfer/models.py b/apps/transfer/models.py new file mode 100644 index 00000000..ccda8cd6 --- /dev/null +++ b/apps/transfer/models.py @@ -0,0 +1,940 @@ +# This is an auto-generated Django model module. +# You'll have to do the following manually to clean this up: +# * Rearrange models' order +# * Make sure each model has one field with primary_key=True +# * Make sure each ForeignKey has `on_delete` set to the desired behavior. +# * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table +# Feel free to rename the models, but don't rename db_table values or field names. +from django.contrib.gis.db import models + +from transfer.mixins import MigrateMixin + + +# models.ForeignKey(ForeignModel, models.DO_NOTHING, blank=True, null=True) + +class Sites(MigrateMixin): + using = 'legacy' + + country_code_2 = models.CharField(max_length=255, blank=True, null=True) + pinterest_page_url = models.CharField(max_length=255, blank=True, null=True) + twitter_page_url = models.CharField(max_length=255, blank=True, null=True) + facebook_page_url = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + contact_email = models.CharField(max_length=255, blank=True, null=True) + config = models.CharField(max_length=3000, blank=True, null=True) + released = models.IntegerField(blank=True, null=True) + instagram_page_url = models.CharField(max_length=255, blank=True, null=True) + ad_config = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'sites' + + +class Features(MigrateMixin): + using = 'legacy' + + slug = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'features' + + +class SiteFeatures(MigrateMixin): + using = 'legacy' + + site = models.ForeignKey('Sites', models.DO_NOTHING, blank=True, null=True) + feature = models.ForeignKey(Features, models.DO_NOTHING, blank=True, null=True) + state = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'site_features' + + +class AwardTypes(MigrateMixin): + using = 'legacy' + + site = models.ForeignKey('Sites', models.DO_NOTHING, blank=True, null=True) + title = models.CharField(max_length=255, blank=True, null=True) + region = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'award_types' + + +class Awards(MigrateMixin): + using = 'legacy' + + profile_id = models.IntegerField(blank=True, null=True) + award_type = models.ForeignKey(AwardTypes, models.DO_NOTHING, blank=True, null=True) + award = models.CharField(max_length=255, blank=True, null=True) + title = models.CharField(max_length=255, blank=True, null=True) + region_code = models.CharField(max_length=255, blank=True, null=True) + year = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + state = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'awards' + + +class Ezuser(MigrateMixin): + using = 'legacy' + + contentobject_id = models.IntegerField(primary_key=True) + email = models.CharField(max_length=150) + login = models.CharField(max_length=150) + password_hash = models.CharField(max_length=50, blank=True, null=True) + password_hash_type = models.IntegerField() + facebook_id = models.BigIntegerField() + # TODO: в legacy нету таблицы 'CadLevel' + + # level = models.ForeignKey('CadLevel', models.DO_NOTHING) + points = models.IntegerField() + publish_fb_activity = models.IntegerField() + + class Meta: + managed = False + db_table = 'ezuser' + + +class Accounts(MigrateMixin): + using = 'legacy' + + uuid = models.CharField(max_length=24) + nickname = models.CharField(max_length=128, blank=True, null=True) + locale = models.CharField(max_length=5, blank=True, null=True) + country_code = models.CharField(max_length=3, blank=True, null=True) + city = models.CharField(max_length=32, blank=True, null=True) + role = models.CharField(max_length=16, blank=True, null=True) + consent_purpose = models.CharField(max_length=255, blank=True, null=True) + consent_at = models.DateTimeField(blank=True, null=True) + last_seen_at = models.DateTimeField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + email = models.CharField(unique=True, max_length=255) + is_admin = models.IntegerField(blank=True, null=True) + ezuser_id = models.IntegerField(blank=True, null=True) + ez_user_id = models.IntegerField(blank=True, null=True) + encrypted_password = models.CharField(max_length=255) + reset_password_token = models.CharField(unique=True, max_length=255, blank=True, null=True) + reset_password_sent_at = models.DateTimeField(blank=True, null=True) + remember_created_at = models.DateTimeField(blank=True, null=True) + sign_in_count = models.IntegerField() + current_sign_in_at = models.DateTimeField(blank=True, null=True) + last_sign_in_at = models.DateTimeField(blank=True, null=True) + current_sign_in_ip = models.CharField(max_length=255, blank=True, null=True) + last_sign_in_ip = models.CharField(max_length=255, blank=True, null=True) + confirmation_token = models.CharField(max_length=255, blank=True, null=True) + confirmed_at = models.DateTimeField(blank=True, null=True) + confirmation_sent_at = models.DateTimeField(blank=True, null=True) + unconfirmed_email = models.CharField(max_length=255, blank=True, null=True) + webpush_subscription = models.CharField(max_length=5000, blank=True, null=True) + + class Meta: + managed = False + db_table = 'accounts' + + +class Profiles(MigrateMixin): + using = 'legacy' + + firstname = models.CharField(max_length=255, blank=True, null=True) + lastname = models.CharField(max_length=255, blank=True, null=True) + gender = models.CharField(max_length=255, blank=True, null=True) + dob = models.DateField(blank=True, null=True) + email = models.CharField(max_length=255, blank=True, null=True) + phone = models.CharField(max_length=255, blank=True, null=True) + site_id = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + unique_key = models.CharField(max_length=255, blank=True, null=True) + account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) + state = models.CharField(max_length=255, blank=True, null=True) + requester_id = models.IntegerField(blank=True, null=True) + available_for_events = models.IntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'profiles' + + +class Cities(MigrateMixin): + using = 'legacy' + + name = models.CharField(max_length=255, blank=True, null=True) + country_code = models.CharField(max_length=3, blank=True, null=True) + country_code_2 = models.CharField(max_length=2, blank=True, null=True) + region_code = models.CharField(max_length=255, blank=True, null=True) + subregion_code = models.CharField(max_length=255, blank=True, null=True) + is_island = models.IntegerField(blank=True, null=True) + zip_code = models.CharField(max_length=9, blank=True, null=True) + situation = models.CharField(max_length=255, blank=True, null=True) + map_ref = models.CharField(max_length=255, blank=True, null=True) + map1 = models.CharField(max_length=255, blank=True, null=True) + map2 = models.CharField(max_length=255, blank=True, null=True) + latitude = models.FloatField(blank=True, null=True) + longitude = models.FloatField(blank=True, null=True) + encima_id = models.IntegerField(blank=True, null=True) + related_city_id = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + index_name = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'cities' + unique_together = (('name', 'region_code', 'country_code'),) + + +class CityNames(MigrateMixin): + using = 'legacy' + + name = models.CharField(max_length=100, blank=True, null=True) + locale = models.CharField(max_length=5, blank=True, null=True) + city = models.ForeignKey(Cities, models.DO_NOTHING, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'city_names' + unique_together = (('city', 'name', 'locale'),) + + +class CityPhotos(MigrateMixin): + using = 'legacy' + + # city_id = models.IntegerField(blank=True, null=True) + city = models.ForeignKey(Cities, models.DO_NOTHING, blank=True, null=True) + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + geometries = models.CharField(max_length=1024, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'city_photos' + + +class Locations(MigrateMixin): + using = 'legacy' + + country_code = models.CharField(max_length=3) + region_code = models.CharField(max_length=3, blank=True, null=True) + subregion_code = models.CharField(max_length=3, blank=True, null=True) + zip_code = models.CharField(max_length=9, blank=True, null=True) + district_name = models.CharField(max_length=255, blank=True, null=True) + longitude = models.DecimalField(max_digits=10, decimal_places=6, blank=True, null=True) + latitude = models.DecimalField(max_digits=10, decimal_places=6, blank=True, null=True) + timezone = models.CharField(max_length=32, blank=True, null=True) + transportation = models.CharField(max_length=255, blank=True, null=True) + address = models.TextField(blank=True, null=True) + city = models.ForeignKey(Cities, models.DO_NOTHING, blank=True, null=True) + map_ref = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'locations' + + +class Collections(MigrateMixin): + using = 'legacy' + + title = models.CharField(max_length=255, blank=True, null=True) + tag_name = models.CharField(max_length=255, blank=True, null=True) + slug = models.CharField(max_length=255, blank=True, null=True) + site_id = models.IntegerField(blank=True, null=True) + active = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + geometries = models.CharField(max_length=1024, blank=True, null=True) + aasm_state = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'collections' + + +# class CollectionEvents(MigrateMixin): +# using = 'legacy' +# +# account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) +# account_collection = models.ForeignKey(AccountCollections, models.DO_NOTHING, blank=True, null=True) +# possible_dates = models.CharField(max_length=255, blank=True, null=True) +# final_date = models.DateTimeField(blank=True, null=True) +# establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# +# class Meta: +# managed = False +# db_table = 'collection_events' + + +# class CollectionEventAvailabilities(MigrateMixin): +# using = 'legacy' +# TODO: collection_event - внешний ключ к CollectionEvents, которая имеет внешний ключ к Accounts + +# collection_event = models.ForeignKey('CollectionEvents', models.DO_NOTHING, blank=True, null=True) +# establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) +# email_address = models.ForeignKey('EmailAddresses', models.DO_NOTHING, blank=True, null=True) +# date = models.DateTimeField(blank=True, null=True) +# state = models.CharField(max_length=255, blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# +# class Meta: +# managed = False +# db_table = 'collection_event_availabilities' + + +class Guides(MigrateMixin): + using = 'legacy' + + title = models.CharField(max_length=255, blank=True, null=True) + vintage = models.IntegerField(blank=True, null=True) + slug = models.CharField(max_length=255, blank=True, null=True) + state = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + site_id = models.IntegerField(blank=True, null=True) + inserter_field = models.CharField(max_length=255, blank=True, null=True) + items_count = models.IntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'guides' + + +class GuideAds(MigrateMixin): + using = 'legacy' + + nb_pages = models.IntegerField(blank=True, null=True) + nb_right_pages = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + guide_ad_node_id = models.IntegerField(blank=True, null=True) + type = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'guide_ads' + + +class GuideFilters(MigrateMixin): + using = 'legacy' + + year = models.TextField(blank=True, null=True) + establishment_type = models.CharField(max_length=255, blank=True, null=True) + countries = models.TextField(blank=True, null=True) + regions = models.TextField(blank=True, null=True) + subregions = models.TextField(blank=True, null=True) + wine_regions = models.TextField(blank=True, null=True) + wine_classifications = models.TextField(blank=True, null=True) + wine_colors = models.TextField(blank=True, null=True) + wine_types = models.TextField(blank=True, null=True) + max_mark = models.FloatField(blank=True, null=True) + min_mark = models.FloatField(blank=True, null=True) + marks_only = models.IntegerField(blank=True, null=True) + locales = models.CharField(max_length=255, blank=True, null=True) + states = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + guide_id = models.IntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'guide_filters' + + +# +# class GuideSections(MigrateMixin): +# using = 'legacy' +# +# type = models.CharField(max_length=255) +# key_name = models.CharField(max_length=255, blank=True, null=True) +# value_name = models.CharField(max_length=255, blank=True, null=True) +# right = models.IntegerField(blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() + + +# class GuideElements(MigrateMixin): +# using = 'legacy' +# +# type = models.CharField(max_length=255) +# establishment = models.ForeignKey(Establishments, models.DO_NOTHING, blank=True, null=True) +# review = models.ForeignKey('Reviews', models.DO_NOTHING, blank=True, null=True) +# review_text = models.ForeignKey('ReviewTexts', models.DO_NOTHING, blank=True, null=True) +# wine_region = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) +# wine = models.ForeignKey('Products', models.DO_NOTHING, blank=True, null=True) +# color = models.CharField(max_length=255, blank=True, null=True) +# order_number = models.IntegerField(blank=True, null=True) +# guide_ad = models.ForeignKey(GuideAds, models.DO_NOTHING, blank=True, null=True) +# city = models.ForeignKey(Cities, models.DO_NOTHING, blank=True, null=True) +# section = models.ForeignKey('GuideSections', models.DO_NOTHING, blank=True, null=True) +# guide_id = models.IntegerField(blank=True, null=True) +# parent_id = models.IntegerField(blank=True, null=True) +# lft = models.IntegerField() +# rgt = models.IntegerField() +# depth = models.IntegerField() +# children_count = models.IntegerField() +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# +# class Meta: +# managed = False +# db_table = 'guide_elements' + + +class Establishments(MigrateMixin): + using = 'legacy' + + name = models.CharField(max_length=255) + index_name = models.CharField(max_length=255, blank=True, null=True) + slug = models.CharField(unique=True, max_length=255, blank=True, null=True) + phone = models.CharField(max_length=255, blank=True, null=True) + fax = models.CharField(max_length=255, blank=True, null=True) + type = models.CharField(max_length=255, blank=True, null=True) + location = models.ForeignKey('Locations', models.DO_NOTHING, blank=True, null=True) + unique_key = models.CharField(max_length=255, blank=True, null=True) + filemaker_id = models.IntegerField(unique=True, blank=True, null=True) + aut_mysql_id = models.IntegerField(unique=True, blank=True, null=True) + fra_encima_id = models.IntegerField(blank=True, null=True) + ca_import_id = models.IntegerField(blank=True, null=True) + ch_import_id = models.IntegerField(blank=True, null=True) + be_ezpublish_id = models.IntegerField(blank=True, null=True) + au_import_id = models.IntegerField(blank=True, null=True) + lux_import_id = models.IntegerField(blank=True, null=True) + hun_import_id = models.IntegerField(blank=True, null=True) + deu_import_id = models.IntegerField(blank=True, null=True) + win_import_id = models.TextField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + state = models.CharField(max_length=255, blank=True, null=True) + manager_revised_at = models.DateTimeField(blank=True, null=True) + cover_id = models.IntegerField(blank=True, null=True) + parent_id = models.IntegerField(blank=True, null=True) + admin_updated_at = models.DateTimeField(blank=True, null=True) + admin_updated_by = models.IntegerField(blank=True, null=True) + company_id = models.IntegerField(blank=True, null=True) + production_type = models.CharField(max_length=3000, blank=True, null=True) + + class Meta: + managed = False + db_table = 'establishments' + + +class Descriptions(MigrateMixin): + using = 'legacy' + + establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) + locale = models.CharField(max_length=5, blank=True, null=True) + text = models.TextField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'descriptions' + + +class Dishes(MigrateMixin): + using = 'legacy' + + name = models.CharField(max_length=255, blank=True, null=True) + price = models.FloatField(blank=True, null=True) + currency = models.CharField(max_length=255, blank=True, null=True) + dish_type = models.CharField(max_length=255, blank=True, null=True) + signature = models.IntegerField(blank=True, null=True) + establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'dishes' + + +class EstablishmentAssets(MigrateMixin): + using = 'legacy' + + establishment = models.ForeignKey('Establishments', models.DO_NOTHING) + account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) + menu_id = models.IntegerField(blank=True, null=True) + type = models.CharField(max_length=64) + scope = models.CharField(max_length=32) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + geometries = models.CharField(max_length=1024, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + attachment_suffix_url = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'establishment_assets' + + +class EstablishmentBacklinks(MigrateMixin): + using = 'legacy' + + establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) + partnership_name = models.CharField(max_length=255, blank=True, null=True) + partnership_icon = models.CharField(max_length=255, blank=True, null=True) + backlink_url = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + type = models.CharField(max_length=255, blank=True, null=True) + starting_date = models.DateField(blank=True, null=True) + expiry_date = models.DateField(blank=True, null=True) + price_per_month = models.DecimalField(max_digits=10, decimal_places=2, blank=True, null=True) + + class Meta: + managed = False + db_table = 'establishment_backlinks' + + +# class EstablishmentCollections(MigrateMixin): +# using = 'legacy' +# +# establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) +# account_collection = models.ForeignKey(AccountCollections, models.DO_NOTHING, blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# +# class Meta: +# managed = False +# db_table = 'establishment_collections' + +class EstablishmentHolidays(MigrateMixin): + using = 'legacy' + + start_date = models.DateField(blank=True, null=True) + end_date = models.DateField(blank=True, null=True) + establishment_id = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'establishment_holidays' + + +class EstablishmentInfos(MigrateMixin): + using = 'legacy' + + establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) + email = models.CharField(max_length=255, blank=True, null=True) + website = models.CharField(max_length=255, blank=True, null=True) + facebook = models.CharField(max_length=255, blank=True, null=True) + twitter = models.CharField(max_length=255, blank=True, null=True) + lafourchette = models.CharField(max_length=255, blank=True, null=True) + pub = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + booking_url = models.CharField(max_length=255, blank=True, null=True) + nb_slot = models.IntegerField(blank=True, null=True) + booking_enabled = models.IntegerField(blank=True, null=True) + guestonline_id = models.IntegerField(blank=True, null=True) + guestonline_auth_token = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'establishment_infos' + + +# class EstablishmentMerchandises(MigrateMixin): +# using = 'legacy' +# +# establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) + +# TODO: модели Merchandises нету в гугл таблице Check Migrations + +# merchandise = models.ForeignKey('Merchandises', models.DO_NOTHING, blank=True, null=True) +# gifted = models.IntegerField(blank=True, null=True) +# quantity = models.IntegerField(blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# +# class Meta: +# managed = False +# db_table = 'establishment_merchandises' + + +class Menus(MigrateMixin): + using = 'legacy' + + establishment = models.ForeignKey(Establishments, models.DO_NOTHING, blank=True, null=True) + name = models.CharField(max_length=255, blank=True, null=True) + timing = models.CharField(max_length=255, blank=True, null=True) + price = models.FloatField(blank=True, null=True) + currency = models.CharField(max_length=255, blank=True, null=True) + drinks = models.CharField(max_length=255, blank=True, null=True) + served_on_offdays = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'menus' + + +class Schedules(MigrateMixin): + using = 'legacy' + + establishment = models.ForeignKey(Establishments, models.DO_NOTHING, blank=True, null=True) + continuous_service = models.CharField(max_length=255, blank=True, null=True) + open_august = models.CharField(max_length=255, blank=True, null=True) + lunch_start = models.TimeField(blank=True, null=True) + lunch_end = models.TimeField(blank=True, null=True) + diner_start = models.TimeField(blank=True, null=True) + diner_end = models.TimeField(blank=True, null=True) + opening_hours = models.CharField(max_length=255, blank=True, null=True) + opening_dates = models.CharField(max_length=255, blank=True, null=True) + timetable = models.CharField(max_length=10000, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'schedules' + + +class MercuryImages(MigrateMixin): + using = 'legacy' + + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + created_at = models.DateTimeField(blank=True, null=True) + updated_at = models.DateTimeField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'mercury_images' + + +class NewsOlds(MigrateMixin): + using = 'legacy' + + title = models.CharField(max_length=255, blank=True, null=True) + body = models.TextField(blank=True, null=True) + slug = models.CharField(max_length=255, blank=True, null=True) + template = models.CharField(max_length=255, blank=True, null=True) + account = models.ForeignKey("Accounts", models.DO_NOTHING, blank=True, null=True) + # site = models.ForeignKey('Sites', models.DO_NOTHING, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + image_file_name = models.CharField(max_length=255, blank=True, null=True) + image_content_type = models.CharField(max_length=255, blank=True, null=True) + image_file_size = models.IntegerField(blank=True, null=True) + image_updated_at = models.DateTimeField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'news_olds' + + +class EmailAddresses(MigrateMixin): + using = 'legacy' + + account = models.ForeignKey("Accounts", models.DO_NOTHING, blank=True, null=True) + partner_notification = models.IntegerField(blank=True, null=True) + ip = models.CharField(max_length=255, blank=True, null=True) + country_code = models.CharField(max_length=3, blank=True, null=True) + city = models.CharField(max_length=255, blank=True, null=True) + locale = models.CharField(max_length=5, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + email = models.CharField(max_length=255, blank=True, null=True) + site_id = models.IntegerField(blank=True, null=True) + state = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'email_addresses' + + +class Reviews(MigrateMixin): + using = 'legacy' + + vintage = models.PositiveIntegerField() + mark = models.FloatField(blank=True, null=True) + favorite = models.IntegerField(blank=True, null=True) + account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True, related_name="account_reviews") + # account_id = models.IntegerField(blank=True, null=True) + establishment = models.ForeignKey(Establishments, models.DO_NOTHING, blank=True, null=True) + visited_at = models.DateField(blank=True, null=True) + created_at = models.DateTimeField() + published_at = models.DateTimeField(blank=True, null=True) + updated_at = models.DateTimeField() + aasm_state = models.CharField(max_length=255, blank=True, null=True) + reviewer_id = models.IntegerField() + priority = models.IntegerField(blank=True, null=True) + # TODO: модель Products в postgres закомментирована + # product = models.ForeignKey("Products", models.DO_NOTHING, blank=True, null=True) + received_at = models.DateTimeField(blank=True, null=True) + reviewer_name = models.CharField(max_length=255, blank=True, null=True) + type = models.CharField(max_length=255, blank=True, null=True) + locked = models.IntegerField(blank=True, null=True) + temporary = models.IntegerField(blank=True, null=True) + last_state_change_at = models.DateTimeField(blank=True, null=True) + editor = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True, related_name="editor_reviews") + + class Meta: + managed = False + db_table = 'reviews' + + +class ReviewTexts(MigrateMixin): + using = 'legacy' + + review = models.ForeignKey('Reviews', models.DO_NOTHING, blank=True, null=True) + # review_id = models.IntegerField(blank=True, null=True) + locale = models.CharField(max_length=5, blank=True, null=True) + text = models.TextField(blank=True, null=True) + updated_by = models.ForeignKey(Accounts, models.DO_NOTHING, db_column='updated_by', blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'review_texts' + + +class Comments(MigrateMixin): + using = 'legacy' + + account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) + establishment = models.ForeignKey('Establishments', models.DO_NOTHING) + parent_id = models.IntegerField(blank=True, null=True) + main_parent_id = models.IntegerField(blank=True, null=True) + comment = models.TextField(blank=True, null=True) + mark = models.DecimalField(max_digits=4, decimal_places=2, blank=True, null=True) + locale = models.CharField(max_length=5) + ip = models.CharField(max_length=40) + state = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + date = models.DateField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'comments' + + +class Pages(MigrateMixin): + using = 'legacy' + + root_title = models.CharField(max_length=255, blank=True, null=True) + site = models.ForeignKey(Sites, models.DO_NOTHING, blank=True, null=True) + account_id = models.IntegerField(blank=True, null=True) + state = models.CharField(max_length=255, blank=True, null=True) + template = models.CharField(max_length=255, blank=True, null=True) + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + attachment_suffix_url = models.TextField(blank=True, null=True) + geometries = models.CharField(max_length=1024, blank=True, null=True) + scheduled_at = models.DateTimeField(blank=True, null=True) + created_at = models.DateTimeField() + published_at = models.DateTimeField(blank=True, null=True) + updated_at = models.DateTimeField() + type = models.CharField(max_length=255, blank=True, null=True) + is_main = models.IntegerField(blank=True, null=True) + related_pages_id = models.TextField(blank=True, null=True) + config = models.CharField(max_length=1000, blank=True, null=True) + + class Meta: + managed = False + db_table = 'pages' + + +class PageTexts(MigrateMixin): + using = 'legacy' + + title = models.CharField(max_length=255, blank=True, null=True) + slug = models.CharField(max_length=255, blank=True, null=True) + body = models.TextField(blank=True, null=True) + locale = models.CharField(max_length=255, blank=True, null=True) + state = models.CharField(max_length=255, blank=True, null=True) + page = models.ForeignKey(Pages, models.DO_NOTHING, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + summary = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'page_texts' + + +class PageCounters(MigrateMixin): + using = 'legacy' + + name = models.CharField(max_length=255, blank=True, null=True) + count = models.IntegerField(blank=True, null=True) + page = models.ForeignKey('Pages', models.DO_NOTHING, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'page_counters' + + +class PageMetadata(MigrateMixin): + using = 'legacy' + + key = models.CharField(max_length=255, blank=True, null=True) + value = models.CharField(max_length=255, blank=True, null=True) + page = models.ForeignKey('Pages', models.DO_NOTHING, blank=True, null=True, related_name='tags') + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'page_metadata' + + +class Ads(MigrateMixin): + using = 'legacy' + + site_id = models.IntegerField(blank=True, null=True) + href = models.CharField(max_length=255, blank=True, null=True) + start_at = models.DateTimeField(blank=True, null=True) + expire_at = models.DateTimeField(blank=True, null=True) + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + geometries = models.CharField(max_length=1024, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'ads' + + +class KeyValueMetadata(MigrateMixin): + using = 'legacy' + + key_name = models.CharField(max_length=255, blank=True, null=True) + value_type = models.CharField(max_length=255, blank=True, null=True) + value_list = models.TextField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + public = models.IntegerField(blank=True, null=True) + site_id = models.IntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'key_value_metadata' + + +class Metadata(MigrateMixin): + using = 'legacy' + + key = models.CharField(max_length=255, blank=True, null=True) + value = models.CharField(max_length=255, blank=True, null=True) + establishment = models.ForeignKey('transfer.Establishments', models.DO_NOTHING, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + key_value_metadatum = models.ForeignKey('transfer.KeyValueMetadata', models.DO_NOTHING, blank=True, null=True) + + class Meta: + managed = False + db_table = 'metadata' + + +class KeyValueMetadatumEstablishments(MigrateMixin): + using = 'legacy' + + name = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'key_value_metadatum_establishments' + + +class KeyValueMetadatumKeyValueMetadatumEstablishments(MigrateMixin): + using = 'legacy' + + key_value_metadatum_id = models.IntegerField(blank=True, null=True) + key_value_metadatum_establishment_id = models.IntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'key_value_metadatum_key_value_metadatum_establishments' + + +# class Products(models.Model): +# establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) +# brand = models.CharField(max_length=255, blank=True, null=True) +# name = models.CharField(max_length=255, blank=True, null=True) +# vintage = models.CharField(max_length=255, blank=True, null=True) +# type = models.CharField(max_length=255, blank=True, null=True) +# unique_key = models.CharField(max_length=255, blank=True, null=True) +# price = models.FloatField(blank=True, null=True) +# average_price_in_shops = models.FloatField(blank=True, null=True) +# fra_encima_id = models.IntegerField(blank=True, null=True) +# wine_sub_region_id = models.IntegerField(blank=True, null=True) +# classification = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) +# wine_region = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) +# wine_type = models.ForeignKey('WineTypes', models.DO_NOTHING, blank=True, null=True) +# wine_color = models.ForeignKey('WineColors', models.DO_NOTHING, blank=True, null=True) +# appellation = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# state = models.CharField(max_length=255, blank=True, null=True) +# wine_style = models.ForeignKey('WineStyles', models.DO_NOTHING, blank=True, null=True) +# village = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) +# vineyard = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) +# yard_classification = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) +# wine_quality = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) +# bottles_produced = models.CharField(max_length=3000, blank=True, null=True) +# deu_import_id = models.IntegerField(blank=True, null=True) +# +# class Meta: +# managed = False +# db_table = 'products' +# +# +# class WineTypes(models.Model): +# name = models.CharField(max_length=255, blank=True, null=True) +# fra_encima_id = models.IntegerField(blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# +# class Meta: +# managed = False +# db_table = 'wine_types' diff --git a/apps/transfer/serializers/__init__.py b/apps/transfer/serializers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/transfer/serializers/account.py b/apps/transfer/serializers/account.py new file mode 100644 index 00000000..5b0e1369 --- /dev/null +++ b/apps/transfer/serializers/account.py @@ -0,0 +1,37 @@ +from rest_framework import serializers +from account.models import User + + +class UserSerializer(serializers.ModelSerializer): + nickname = serializers.CharField() + email = serializers.CharField() + confirmed_at = serializers.DateTimeField() + id = serializers.CharField() + + class Meta: + model = User + + fields = ( + "id", + "nickname", + "email", + "confirmed_at" + ) + + def validate(self, data): + data["old_id"] = data.pop("id") + data["username"] = self.get_username(data) + data["email_confirmed"] = self.get_email_confirmed(data) + data.pop("nickname") + data.pop("confirmed_at") + return data + + def create(self, validated_data): + # использовать get_or_create + User.objects.create(**validated_data) + + def get_email_confirmed(self, obj): + return True + + def get_username(self, obj): + return obj["email"] diff --git a/apps/transfer/serializers/advertisement.py b/apps/transfer/serializers/advertisement.py new file mode 100644 index 00000000..2e3a488f --- /dev/null +++ b/apps/transfer/serializers/advertisement.py @@ -0,0 +1,39 @@ +from rest_framework import serializers +from advertisement.models import Advertisement +import yaml + + +class AdvertisementSerializer(serializers.ModelSerializer): + href = serializers.CharField() + geometries = serializers.CharField(max_length=1024) + + class Meta: + model = Advertisement + fields = ( + "href", + "geometries" + ) + + def validate(self, data): + data["url"] = data["href"] + data["width"] = self.get_width(data["geometries"]) + data["height"] = self.get_height(data["geometries"]) + data.pop("href") + data.pop("geometries") + return data + + def create(self, validated_data): + return Advertisement.objects.create(**validated_data) + + def get_width(self, data): + data = self.parse_geometries(data) + return int(float(data["width"])) + + def get_height(self, data): + data = self.parse_geometries(data) + return int(float(data["height"])) + + def parse_geometries(self, geo_str): + clear_str = "!ruby/object:Paperclip::Geometry" + content_dict = yaml.safe_load(geo_str.replace(clear_str, '')) + return content_dict[':original'] \ No newline at end of file diff --git a/apps/transfer/serializers/comments.py b/apps/transfer/serializers/comments.py new file mode 100644 index 00000000..584370e6 --- /dev/null +++ b/apps/transfer/serializers/comments.py @@ -0,0 +1,64 @@ +from rest_framework import serializers +from comment.models import Comment, User +from establishment.models import Establishment + + +class CommentSerializer(serializers.ModelSerializer): + id = serializers.IntegerField() + comment = serializers.CharField() + mark = serializers.DecimalField(max_digits=4, decimal_places=2) + locale = serializers.CharField() + account_id = serializers.IntegerField() + establishment_id = serializers.CharField() + + class Meta: + model = Comment + fields = ( + "id", + "comment", + "mark", + "locale", + "account_id", + "establishment_id" + ) + + def validate(self, data): + data = self.set_old_id(data) + data = self.set_text(data) + data = self.set_mark(data) + data = self.set_establishment(data) + data = self.set_account(data) + return data + + def set_text(self, data): + data['text'] = data.pop('comment') + return data + + def set_mark(self, data): + if data['mark'] < 0: + data['mark'] = data['mark'] * -1 + return data + + def set_account(self, data): + try: + data['account'] = User.objects.filter(old_id=data['account_id']).first() + except User.DoesNotExist as e: + raise ValueError(f"User account not found with {data}: {e}") + + del(data['account_id']) + + return data + + def set_establishment(self, data): + try: + data['establishment'] = Establishment.objects.filter(old_id=data['account_id']).first() + except Establishment.DoesNotExist as e: + raise ValueError(f"Establishment not found with {data}: {e}") + + del(data['establishment_id']) + + return data + + def set_old_id(self, data): + data['old_id'] = data.pop("id") + return data diff --git a/apps/transfer/serializers/establishment.py b/apps/transfer/serializers/establishment.py new file mode 100644 index 00000000..7c8ce97c --- /dev/null +++ b/apps/transfer/serializers/establishment.py @@ -0,0 +1,156 @@ +from django.core.exceptions import MultipleObjectsReturned, ValidationError +from django.db import transaction +from rest_framework import serializers + +from establishment.models import Establishment, ContactEmail, ContactPhone, EstablishmentType +from location.models import Address +from timetable.models import Timetable +from utils.legacy_parser import parse_legacy_schedule_content +from utils.slug_generator import generate_unique_slug +from pytz import timezone as ptz + + +class EstablishmentSerializer(serializers.ModelSerializer): + slug = serializers.CharField(allow_null=True, allow_blank=True) + type = serializers.CharField() + description = serializers.DictField( + allow_null=True, + child=serializers.CharField(allow_null=True), + ) + schedules = serializers.CharField(allow_null=True, allow_blank=True) + location = serializers.IntegerField(allow_null=True) + email = serializers.CharField(allow_null=True, allow_blank=True) + phone = serializers.CharField(allow_null=True, allow_blank=True) + website = serializers.CharField(allow_null=True, allow_blank=True) + facebook = serializers.CharField(allow_null=True, allow_blank=True) + twitter = serializers.CharField(allow_null=True, allow_blank=True) + booking = serializers.CharField(allow_null=True, allow_blank=True) + state = serializers.CharField(allow_null=True) + tz = serializers.CharField() + created = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') + + class Meta: + model = Establishment + fields = ( + 'created', + 'old_id', # + + 'name', # + + 'name_translated', # + + 'tz', # + + 'website', # + + 'facebook', # + + 'twitter', # + + 'lafourchette', # + + 'booking', # + + 'type', # + см в JIRA + 'slug', # + сгенерировать уникальный слаг + 'description', # + (разобрал в transfer_data) + 'schedules', # + разобрать RUBY словать (2 варианта) + 'location', # + получить новые объекты Address по old_id + 'email', # + создать объект для ContactEmail + 'phone', # + создать объект для ContactPhone + 'state', # + создать объект для ContactPhone + ) + + def validate(self, data): + data.update({ + 'slug': generate_unique_slug(Establishment, data['slug'] if data['slug'] else data['name']), + 'address_id': self.get_address(data['location']), + 'establishment_type_id': self.get_type(data), + 'is_publish': data.get('state') == 'published', + }) + data['tz'] = ptz(data['tz']) + data.pop('location') + data.pop('type') + data.pop('state') + return data + + @transaction.atomic + def create(self, validated_data): + email = validated_data.pop('email') + phone = validated_data.pop('phone') + schedules = validated_data.pop('schedules') + + establishment = Establishment.objects.create(**validated_data) + if email: + ContactEmail.objects.get_or_create( + email=email, + establishment=establishment, + ) + if phone: + ContactPhone.objects.get_or_create( + phone=phone, + establishment=establishment, + ) + if schedules: + new_schedules = self.get_schedules(schedules) + for schedule in new_schedules: + establishment.schedule.add(schedule) + establishment.save() + + return establishment + + @staticmethod + def get_address(address): + address = Address.objects.filter(old_id=address).first() + if address: + return address.id + return None + # return None + + @staticmethod + def get_type(data): + types = { + 'Restaurant': EstablishmentType.RESTAURANT, + 'Shop': EstablishmentType.ARTISAN, + } + obj, _ = EstablishmentType.objects.get_or_create(index_name=types[data['type']]) + return obj.id + + @staticmethod + def get_schedules(schedules): + result = [] + legacy_dict = parse_legacy_schedule_content(schedules) + + weekdays = { + 'su': Timetable.SUNDAY, + 'mo': Timetable.MONDAY, + 'tu': Timetable.THURSDAY, + 'we': Timetable.WEDNESDAY, + 'th': Timetable.THURSDAY, + 'fr': Timetable.FRIDAY, + 'sa': Timetable.SATURDAY, + } + + for key, val in legacy_dict.items(): + payload = { + 'weekday': weekdays[key], + 'lunch_start': None, + 'lunch_end': None, + 'dinner_start': None, + 'dinner_end': None, + 'opening_at': None, + 'closed_at': None, + } + if val.get('morning'): + payload.update({ + 'lunch_start': val['morning'].get('start'), + 'lunch_end': val['morning'].get('end'), + }) + if val.get('afternoon'): + payload.update({ + 'dinner_start': val['afternoon'].get('start'), + 'dinner_end': val['afternoon'].get('end'), + }) + + try: + obj, _ = Timetable.objects.get_or_create(**payload) + except ValidationError: + obj = None + except MultipleObjectsReturned: + obj = Timetable.objects.filter(**payload).first() + + if obj: + result.append(obj) + + return result diff --git a/apps/transfer/serializers/gallery.py b/apps/transfer/serializers/gallery.py new file mode 100644 index 00000000..273dd703 --- /dev/null +++ b/apps/transfer/serializers/gallery.py @@ -0,0 +1,15 @@ +from rest_framework import serializers +from gallery.models import Image + + +class ImageSerializer(serializers.ModelSerializer): + attachment_file_name = serializers.CharField(source="image") + + class Meta: + model = Image + fields = ( + "attachment_file_name", + ) + + def create(self, validated_data): + return Image.objects.create(**validated_data) diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py new file mode 100644 index 00000000..d4ea51cc --- /dev/null +++ b/apps/transfer/serializers/location.py @@ -0,0 +1,268 @@ +from rest_framework import serializers +from location.models import Country, Region, City, Address +from django.contrib.gis.geos import Point +from django.contrib.gis.geos import GEOSGeometry +import json + + +class CountrySerializer(serializers.ModelSerializer): + country_code_2 = serializers.CharField() + id = serializers.IntegerField() + + class Meta: + model = Country + fields = ( + "id", + "country_code_2", + ) + + def validate(self, data): + data["code"] = self.get_country_code(data) + del(data['country_code_2']) + + data['old_id'] = data.pop('id') + + return data + + def create(self, validated_data): + # Some countries already in database + try: + country = Country.objects.get(code=validated_data['code']) + except Country.DoesNotExist: + country = Country.objects.create(**validated_data) + return country + + def get_country_code(self, obj): + return obj.get("country_code_2") + + +class RegionSerializer(serializers.ModelSerializer): + region_code = serializers.CharField() + subregion_code = serializers.CharField(allow_null=True, allow_blank=True) + country_code_2 = serializers.CharField() + id = serializers.IntegerField() + + class Meta: + model = Region + fields = ( + "region_code", + "country_code_2", + "subregion_code", + "id" + ) + + def validate(self, data): + data = self.set_old_id(data) + data = self.set_code(data) + data = self.set_country(data) + return data + + def create(self, validated_data): + # Some regions may be already in database + try: + region = Region.objects.get(old_id=validated_data['old_id']) + except Region.DoesNotExist: + region = Region.objects.create(**validated_data) + except Exception as e: + raise ValueError(f"REGION ERROR: {validated_data}: {e}") + return region + + def set_code(self, data): + print(data) + if "subregion_code" in data and data["subregion_code"] is not None and data["subregion_code"].strip() != "": + try: + parent_region = Region.objects.get(code=str(data['region_code'])) + except Exception as e: + raise ValueError(f"Parent region error with {data}: {e}") + + data['parent_region'] = parent_region + data['code'] = data.pop('subregion_code') + del(data['region_code']) + else: + data['code'] = data.pop('region_code') + del(data['subregion_code']) + + return data + + def set_country(self, data): + try: + country = Country.objects.get(code=data['country_code_2']) + except Exception as e: + raise ValueError(f"Country error with {data}: {e}") + + data["country"] = country + del(data['country_code_2']) + + return data + + def set_old_id(self, data): + data['old_id'] = data.pop("id") + return data + + +class CitySerializer(serializers.ModelSerializer): + country_code_2 = serializers.CharField() + region_code = serializers.CharField() + subregion_code = serializers.CharField(allow_null=True, allow_blank=True) + zip_code = serializers.CharField(allow_null=True, allow_blank=True) + is_island = serializers.IntegerField(allow_null=True) + name = serializers.CharField() + id = serializers.IntegerField() + + class Meta: + model = City + fields = ( + "country_code_2", + "region_code", + "subregion_code", + "zip_code", + "is_island", + "name", + "id" + ) + + def validate(self, data): + data = self.set_old_id(data) + data = self.set_relations(data) + data = self.set_is_island(data) + data = self.set_code(data) + data = self.set_zip_code(data) + return data + + def create(self, validated_data): + return City.objects.create(**validated_data) + + def set_is_island(self, data): + data['is_island'] = True if "is_island" in data \ + and data['is_island'] is not None \ + and data['is_island'] > 0 \ + else False + return data + + def set_code(self, data): + data['code'] = data.pop('region_code') + return data + + def set_relations(self, data): + try: + region = Region.objects.filter(code=data['region_code']).first() + except Region.DoesNotExist as e: + try: + region = Region.objects.filter(code=data['subregion_code']).first() + except Region.DoesNotExist as e: + raise ValueError(f"Region not found with {data}: {e}") + + data['region'] = region + del(data['subregion_code']) + + try: + country = Country.objects.get(code=data['country_code_2']) + except Country.DoesNotExist as e: + raise ValueError(f"Country not found with {data}: {e}") + + data['country'] = country + del(data['country_code_2']) + + return data + + def set_zip_code(self, data): + data['postal_code'] = data.pop('zip_code') + if data['postal_code'] is None: + data['postal_code'] = "" + return data + + def set_old_id(self, data): + data['old_id'] = data.pop('id') + return data + + +class AddressSerializer(serializers.ModelSerializer): + id = serializers.IntegerField() + city_id = serializers.IntegerField() + zip_code = serializers.CharField(allow_null=True, allow_blank=True) + latitude = serializers.DecimalField(max_digits=10, decimal_places=6, allow_null=True) + longitude = serializers.DecimalField(max_digits=10, decimal_places=6, allow_null=True) + address = serializers.CharField() + + class Meta: + model = Address + fields = ( + "id", + "city_id", + "zip_code", + "latitude", + "longitude", + "address" + ) + + def validate(self, data): + data = self.set_old_id(data) + data = self.set_address(data) + data = self.set_postal_code(data) + data = self.set_city(data) + data = self.set_point(data) + return data + + def create(self, validated_data): + return Address.objects.create(**validated_data) + + def set_old_id(self, data): + data['old_id'] = data.pop("id") + return data + + def set_postal_code(self, data): + data['postal_code'] = data.pop('zip_code', None) + if data['postal_code'] is None: + data['postal_code'] = "" + return data + + def set_city(self, data): + try: + city = City.objects.filter(old_id=data['city_id']).first() + except City.DoesNotExist as e: + raise ValueError(f"City not found with {data}: {e}") + + data['city'] = city + del(data['city_id']) + return data + + def set_address(self, data): + address_list = data.pop('address').split(' ') + is_first_street = False + data['street_name_1'] = [] + data['street_name_2'] = [] + while len(address_list) > 0: + address_part = address_list.pop() + try: + address_part = int(address_part) + data['number'] = address_part + is_first_street = True + except: + if is_first_street: + data['street_name_1'].append(address_part) + else: + data['street_name_2'].append(address_part) + + data['street_name_1'] = " ".join(data['street_name_1']) + data['street_name_2'] = " ".join(data['street_name_2']) + if "number" not in data: + data['number'] = 0 + + return data + + def set_point(self, data): + if data['latitude'] is not None and data['longitude'] is not None: + data['coordinates'] = Point(float(data['longitude']), float(data['latitude'])) + # data['coordinates'] = GEOSGeometry( + # json.dumps({ + # "type": "Point", + # "coordinates": [data['longitude'], data['latitude']] + # }, ensure_ascii=False, default=str) + # ) + else: + data['coordinates'] = None + + del(data['latitude']) + del(data['longitude']) + + return data diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py new file mode 100644 index 00000000..4dc7d913 --- /dev/null +++ b/apps/transfer/serializers/news.py @@ -0,0 +1,128 @@ +from rest_framework import serializers + +from gallery.models import Image +from location.models import Country +from news.models import News, NewsGallery +from tag.models import Tag +from transfer.models import PageMetadata +from utils.legacy_parser import parse_legacy_news_content +from utils.slug_generator import generate_unique_slug + + +class NewsSerializer(serializers.Serializer): + id = serializers.IntegerField() + tag_cat_id = serializers.IntegerField() + news_type_id = serializers.IntegerField() + news_title = serializers.CharField() + title = serializers.CharField() + summary = serializers.CharField(allow_null=True, allow_blank=True) + body = serializers.CharField(allow_null=True) + created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') + slug = serializers.CharField() + state = serializers.CharField() + template = serializers.CharField() + country_code = serializers.CharField(allow_null=True) + locale = serializers.CharField() + image = serializers.CharField() + tags = serializers.CharField(allow_null=True) + + def create(self, validated_data): + + payload = { + 'old_id': validated_data['id'], + 'news_type_id': validated_data['news_type_id'], + 'title': {validated_data['locale']: validated_data['news_title']}, + 'subtitle': self.get_subtitle(validated_data), + 'description': self.get_description(validated_data), + 'start': validated_data['created_at'], + 'slug': generate_unique_slug(News, validated_data['slug']), + 'state': self.get_state(validated_data), + 'template': self.get_template(validated_data), + 'country': self.get_country(validated_data), + } + obj = News.objects.create(**payload) + + tags = self.get_tags(validated_data) + for tag in tags: + obj.tags.add(tag) + obj.save() + + self.make_gallery(validated_data, obj) + return obj + + @staticmethod + def make_gallery(data, obj): + if not data['image'] or data['image'] == 'default/missing.png': + return + + img = Image.objects.create( + image=data['image'], + title=data['news_title'], + ) + NewsGallery.objects.create( + news=obj, + image=img, + is_main=True, + ) + + @staticmethod + def get_tags(data): + results = [] + if not data['tags']: + return results + + meta_ids = (int(_id) for _id in data['tags'].split(',')) + tags = PageMetadata.objects.filter( + id__in=meta_ids, + key='tag', + value__isnull=False, + ) + for old_tag in tags: + tag, _ = Tag.objects.get_or_create( + category_id=data['tag_cat_id'], + label={data['locale']: old_tag.value}, + ) + results.append(tag) + return results + + @staticmethod + def get_description(data): + if data['body']: + content = parse_legacy_news_content(data['body']) + return {data['locale']: content} + return None + + @staticmethod + def get_state(data): + states = { + 'new': News.WAITING, + 'published': News.PUBLISHED, + 'hidden': News.HIDDEN, + 'published_exclusive': News.PUBLISHED_EXCLUSIVE, + 'scheduled_exclusively': News.WAITING, + } + return states.get(data['state'], News.WAITING) + + @staticmethod + def get_template(data): + templates = { + 'main': News.MAIN, + 'main.pdf.erb': News.MAIN_PDF_ERB, + } + return templates.get(data['template'], News.MAIN) + + @staticmethod + def get_country(data): + return Country.objects.filter(code__iexact=data['country_code']).first() + + @staticmethod + def get_title(data): + return {data['locale']: data['title']} + + @staticmethod + def get_subtitle(data): + if data.get('summary'): + content = {data['locale']: data['summary']} + else: + content = {data['locale']: data['title']} + return content diff --git a/apps/transfer/serializers/notification.py b/apps/transfer/serializers/notification.py new file mode 100644 index 00000000..c179dd2a --- /dev/null +++ b/apps/transfer/serializers/notification.py @@ -0,0 +1,35 @@ +from rest_framework import serializers +from notification.models import Subscriber + + +class SubscriberSerializer(serializers.ModelSerializer): + email = serializers.CharField() + locale = serializers.CharField(allow_null=True) + country_code = serializers.CharField(allow_null=True) + + class Meta: + model = Subscriber + fields = ( + "email", + "locale", + "country_code" + ) + + def validate(self, data): + data["email"] = self.get_email(data) + data["locale"] = self.get_locale(data) + data["country_code"] = self.get_country_code(data) + return data + + def create(self, validated_data): + inst, created = Subscriber.objects.get_or_create(**validated_data) + return inst + + def get_email(self, obj): + return obj["email"] + + def get_locale(self, obj): + return obj["locale"] + + def get_country_code(self, obj): + return obj["country_code"] diff --git a/apps/transfer/serializers/partner.py b/apps/transfer/serializers/partner.py new file mode 100644 index 00000000..094418c6 --- /dev/null +++ b/apps/transfer/serializers/partner.py @@ -0,0 +1,15 @@ +from rest_framework import serializers +from partner.models import Partner + + +class PartnerSerializer(serializers.ModelSerializer): + backlink_url = serializers.CharField(source="url") + + class Meta: + model = Partner + fields = ( + "backlink_url", + ) + + def create(self, validated_data): + return Partner.objects.create(**validated_data) diff --git a/apps/transfer/serializers/plate.py b/apps/transfer/serializers/plate.py new file mode 100644 index 00000000..7d5eb99b --- /dev/null +++ b/apps/transfer/serializers/plate.py @@ -0,0 +1,55 @@ +from rest_framework import serializers + +from establishment.models import Menu, Plate, Establishment +from main.models import Currency +from utils.constants import CODE_LOCALES + + +class PlateSerializer(serializers.Serializer): + name = serializers.CharField() + price = serializers.DecimalField(decimal_places=2, max_digits=10) + currency = serializers.CharField() + dish_type = serializers.CharField() + country_code = serializers.CharField() + establishment_id = serializers.IntegerField() + signature = serializers.IntegerField(allow_null=True) + + def create(self, validated_data): + establishment = Establishment.objects.filter(old_id=validated_data['establishment_id']).first() + if not establishment: + return + return Plate.objects.create(**self.get_plate_data(validated_data, establishment.id)) + + def get_plate_data(self, validated_data, est_id): + locale = CODE_LOCALES.get(validated_data['country_code'], 'en-GB') + payload = { + 'name': {locale: validated_data['name']}, + 'price': validated_data['price'], + 'currency_id': self.get_currency(validated_data), + 'menu_id': self.get_menu(validated_data, est_id), + 'is_signature_plate': bool(validated_data['signature']), + } + return payload + + @staticmethod + def get_menu(validated_data, est_id): + payload = { + 'establishment_id': est_id, + 'category': {'en-GB': validated_data['dish_type']}, + } + menu, _ = Menu.objects.get_or_create(**payload) + return menu.id + + @staticmethod + def get_currency(validated_data): + try: + currency = Currency.objects.get( + slug=validated_data['currency'], + sign='$', + ) + except Currency.DoesNotExist: + currency = Currency.objects.create( + slug=validated_data['currency'], + sign='$', + ) + return currency.id diff --git a/apps/transfer/serializers/recipe.py b/apps/transfer/serializers/recipe.py new file mode 100644 index 00000000..11698ba1 --- /dev/null +++ b/apps/transfer/serializers/recipe.py @@ -0,0 +1,55 @@ +from rest_framework import serializers +from recipe.models import Recipe +from utils.legacy_parser import parse_legacy_news_content + + +class RecipeSerializer(serializers.ModelSerializer): + locale = serializers.CharField() + body = serializers.CharField(allow_null=True) + title = serializers.CharField() + state = serializers.CharField() + created_at = serializers.DateTimeField(source="published_at", format='%m-%d-%Y %H:%M:%S') + + class Meta: + model = Recipe + fields = ( + "body", + "title", + "state", + "created_at", + 'locale', + ) + + def validate(self, data): + data["state"] = self.get_state(data) + data["title"] = self.get_title(data) + data["description"] = self.get_description(data) + data.pop("body") + data.pop("locale") + return data + + def create(self, validated_data): + return Recipe.objects.create(**validated_data) + + def get_state(self, obj): + if obj["state"] == "published": + return Recipe.PUBLISHED + elif obj["state"] == "hidden": + return Recipe.HIDDEN + elif obj["state"] == "published_exclusive": + return Recipe.PUBLISHED_EXCLUSIVE + else: + return Recipe.WAITING + + def get_title(self, obj): + # tit = obj.get("title") + # return {"en-GB": tit} + return {obj['locale']: obj['title']} + + def get_description(self, obj): + # desc = obj.get("body") + # return {"en-GB": desc} + content = None + if obj['body']: + content = parse_legacy_news_content(obj['body']) + return {obj['locale']: content} diff --git a/apps/transfer/serializers/reviews.py b/apps/transfer/serializers/reviews.py new file mode 100644 index 00000000..024ea033 --- /dev/null +++ b/apps/transfer/serializers/reviews.py @@ -0,0 +1,213 @@ +from rest_framework import serializers +from review.models import Review +from account.models import User +from translation.models import Language +from establishment.models import Establishment + + +class ReviewSerializer(serializers.ModelSerializer): + id = serializers.IntegerField() + reviewer_id = serializers.IntegerField() + vintage = serializers.IntegerField() + published = serializers.DateTimeField() + published_at = serializers.DateTimeField(allow_null=True) + establishment_id = serializers.IntegerField() + text = serializers.CharField() + locale = serializers.CharField() + aasm_state = serializers.CharField(allow_null=True) + + class Meta: + model = Review + fields = ( + "id", "reviewer_id", "published", "vintage", + "establishment_id", "text", "locale", + "published_at", "aasm_state" + ) + + def validate(self, data): + data = self.set_old_id(data) + data = self.set_published_at(data) + data = self.set_reviewer(data) + data = self.set_establishment(data) + data = self.set_language(data) + data = self.set_published(data) + return data + + def create(self, validated_data): + try: + return Review.objects.create(**validated_data) + except Exception as e: + raise ValueError(f"Error creating review with {validated_data}: {e}") + + def set_old_id(self, data): + data['old_id'] = data.pop("id") + return data + + def set_reviewer(self, data): + try: + data['reviewer'] = User.objects.get(old_id=data['reviewer_id']) + except User.DoesNotExist as e: + raise ValueError(f"Cannot find reviewer with {data}: {e}") + del(data['reviewer_id']) + return data + + def set_establishment(self, data): + try: + data['content_object'] = Establishment.objects.get(old_id=data.pop('establishment_id')) + except Establishment.DoesNotExist as e: + raise ValueError(f"Cannot find review establishment with {data}: {e}") + return data + + def set_language(self, data): + try: + data['language'] = Language.objects.get(locale=data['locale']) + except Language.DoesNotExist as e: + raise ValueError(f"Cannot find language with {data}: {e}") + + del(data['locale']) + + return data + + def set_published(self, data): + data['published_at'] = data.pop("published") + return data + + def set_published_at(self, data): + if "aasm_state" in data and data['aasm_state'] is not None and data['aasm_state'] == "published": + data['status'] = Review.READY + del(data['aasm_state']) + return data + + +class LanguageSerializer(serializers.ModelSerializer): + + LANGUAGES = { + "be-BE": "Belarusian - Belarusia", + "el-GR": "Greek - Greek", + "he-IL": "Israel - Hebrew", + "hr-HR": "Croatia - Croatian", + "hu-HU": "Hangarian - Hungary", + "ja-JP": "Japanese - Japan", + "ka-GE": "Georgian - Georgia", + "pl-PL": "Polish - Poland", + "ro-RO": "Romainian - Romania", + "ru-RU": "Russian - Russia", + "sl-SI": "Slovenian - Slovenia", + "ar-DZ": "Arabic - Algeria", + "ar-BH": "Arabic - Bahrain", + "ar-EG": "Arabic - Egypt", + "ar-IQ": "Arabic - Iraq", + "ar-JO": "Arabic - Jordan", + "ar-KW": "Arabic - Kuwait", + "ar-LB": "Arabic - Lebanon", + "ar-LY": "Arabic - Libya", + "ar-MA": "Arabic - Morocco", + "ar-OM": "Arabic - Oman", + "ar-QA": "Arabic - Qatar", + "ar-SA": "Arabic - Saudi Arabia", + "ar-SY": "Arabic - Syria", + "ar-TN": "Arabic - Tunisia", + "ar-AE": "Arabic - United Arab Emirates", + "ar-YE": "Arabic - Yemen", + "az-AZ": "Azeri - Cyrillic", + "zh-CN": "Chinese - China", + "zh-HK": "Chinese - Hong Kong SAR", + "zh-MO": "Chinese - Macau SAR", + "zh-SG": "Chinese - Singapore", + "zh-TW": "Chinese - Taiwan", + "nl-BE": "Dutch - Belgium", + "nl-NL": "Dutch - Netherlands", + "en-AU": "English - Australia", + "en-BZ": "English - Belize", + "en-CA": "English - Canada", + "en-CB": "English - Caribbean", + "en-GB": "English - Great Britain", + "en-IN": "English - India", + "en-IE": "English - Ireland", + "en-JM": "English - Jamaica", + "en-NZ": "English - New Zealand", + "en-PH": "English - Phillippines", + "en-ZA": "English - Southern Africa", + "en-TT": "English - Trinidad", + "en-US": "English - United States", + "fr-BE": "French - Belgium", + "fr-CA": "French - Canada", + "fr-FR": "French - France", + "fr-LU": "French - Luxembourg", + "fr-CH": "French - Switzerland", + "fr-MA": "French - Morocco", + "gd-IE": "Gaelic - Ireland", + "de-AT": "German - Austria", + "de-DE": "German - Germany", + "de-LI": "German - Liechtenstein", + "de-LU": "German - Luxembourg", + "de-CH": "German - Switzerland", + "it-IT": "Italian - Italy", + "it-CH": "Italian - Switzerland", + "ms-BN": "Malay - Brunei", + "ms-MY": "Malay - Malaysia", + "no-NO": "Norwegian - Bokml", + "pt-BR": "Portuguese - Brazil", + "pt-PT": "Portuguese - Portugal", + "ro-MO": "Romanian - Moldova", + "ru-MO": "Russian - Moldova", + "sr-SP": "Serbian - Cyrillic", + "es-AR": "Spanish - Argentina", + "es-BO": "Spanish - Bolivia", + "es-CL": "Spanish - Chile", + "es-CO": "Spanish - Colombia", + "es-CR": "Spanish - Costa Rica", + "es-DO": "Spanish - Dominican Republic", + "es-EC": "Spanish - Ecuador", + "es-SV": "Spanish - El Salvador", + "es-GT": "Spanish - Guatemala", + "es-HN": "Spanish - Honduras", + "es-MX": "Spanish - Mexico", + "es-NI": "Spanish - Nicaragua", + "es-PA": "Spanish - Panama", + "es-PY": "Spanish - Paraguay", + "es-PE": "Spanish - Peru", + "es-PR": "Spanish - Puerto Rico", + "es-ES": "Spanish - Spain (Traditional)", + "es-UY": "Spanish - Uruguay", + "es-VE": "Spanish - Venezuela", + "sv-FI": "Swedish - Finland", + "sv-SE": "Swedish - Sweden", + "uz-UZ": "Uzbek - Cyrillic", + } + + id = serializers.CharField() + locale = serializers.CharField() + + class Meta: + model = Language + fields = ("locale", "id") + + def validate(self, data): + data = self.set_old_id(data) + data = self.set_locale(data) + return data + + def create(self, validated_data): + try: + locale = Language.objects.filter(locale=validated_data['locale']).first() + if locale is None: + raise Language.DoesNotExist + locale.old_id = validated_data['old_id'] + locale.save() + except Language.DoesNotExist: + locale = Language.objects.create(**validated_data) + + return locale + + def set_locale(self, data): + data['locale'] = data['locale'].replace("_", "-") + try: + data['title'] = self.LANGUAGES[data['locale']] + except Exception as e: + raise ValueError(f"{data}: {e}") + return data + + def set_old_id(self, data): + data['old_id'] = data.pop("id") + return data diff --git a/apps/transfer/tests.py b/apps/transfer/tests.py new file mode 100644 index 00000000..593d2501 --- /dev/null +++ b/apps/transfer/tests.py @@ -0,0 +1,14 @@ +from django.test import TestCase +# from django.core.management import call_command +# +# +# class NewsTransferTestCase(TestCase): +# command = "-n" +# +# databases = {'default', 'legacy'} +# +# def test_transfer(self): +# args = [self.command] +# kwargs = {} +# +# call_command('transfer', *args, **kwargs) diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py new file mode 100644 index 00000000..b83d5b2b --- /dev/null +++ b/apps/transfer/utils.py @@ -0,0 +1,18 @@ +from os.path import exists +from importlib.machinery import SourceFileLoader +from django.apps import apps + + +def transfer_objects(data_type): + + for app in apps.get_app_configs(): + if exists(f"{app.path}/transfer_data.py"): + card_module = SourceFileLoader("transfer", f"{app.path}/transfer_data.py").load_module() + if not hasattr(card_module, "data_types") or not isinstance(card_module.data_types, dict) or len(card_module.data_types) < 1: + continue + + for module_data_type, transfer_funcs in card_module.data_types.items(): + if data_type == module_data_type: + for transfer_func in transfer_funcs: + print(f"========================== FUNCTION {transfer_func.__name__} ================================") + transfer_func() diff --git a/apps/transfer/views.py b/apps/transfer/views.py new file mode 100644 index 00000000..91ea44a2 --- /dev/null +++ b/apps/transfer/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/apps/translation/migrations/0006_language_old_id.py b/apps/translation/migrations/0006_language_old_id.py new file mode 100644 index 00000000..ea7d0ce8 --- /dev/null +++ b/apps/translation/migrations/0006_language_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-30 05:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('translation', '0005_auto_20191021_1201'), + ] + + operations = [ + migrations.AddField( + model_name='language', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/translation/models.py b/apps/translation/models.py index cb0729ea..20b00233 100644 --- a/apps/translation/models.py +++ b/apps/translation/models.py @@ -25,6 +25,8 @@ class Language(models.Model): locale = models.CharField(max_length=10, verbose_name=_('Locale identifier')) + old_id = models.IntegerField(null=True, blank=True, default=None) + objects = LanguageQuerySet.as_manager() class Meta: diff --git a/apps/utils/admin.py b/apps/utils/admin.py new file mode 100644 index 00000000..fd5e353f --- /dev/null +++ b/apps/utils/admin.py @@ -0,0 +1,35 @@ +"""Mixins for admin models.""" +from django.db.models import ForeignKey + + +class BaseModelAdminMixin: + """ + Class that overridden ModelAdmin and adds to readonly_fields attr + persisted fields like created_by, modified_by. + """ + + _PERSISTENT_READ_ONLY_FIELDS = ['created_by', 'modified_by'] + + def _get_fk_field_names(self, fields: iter): + """ + Return an iterable object which contains FK model fields. + :param fields: iterable + :return: iterable + """ + foreign_key_fields = [] + for field in fields: + if isinstance(field, ForeignKey): + foreign_key_fields.append(field.name) + return foreign_key_fields + + def get_readonly_fields(self, request, obj=None): + """ + Hook for specifying custom readonly fields. + """ + _readonly_fields = list(self.readonly_fields) + fk_field_names = self._get_fk_field_names(self.model._meta.fields) + + for field_name in fk_field_names: + if field_name in self._PERSISTENT_READ_ONLY_FIELDS: + _readonly_fields.append(field_name) + return tuple(_readonly_fields) diff --git a/apps/utils/constants.py b/apps/utils/constants.py new file mode 100644 index 00000000..fe24c008 --- /dev/null +++ b/apps/utils/constants.py @@ -0,0 +1,26 @@ +CODE_LOCALES = { + 'AAA': 'en-GB', + 'AUS': 'en-AU', # Австралия + 'AUT': 'de-AT', # Австрия + 'BEL': 'de-BE', # Бельгия + 'BRA': 'pt-BR', # Бразилия + 'CAN': 'fr-CA', # Канада + 'DEU': 'da-DE', # Германия + 'FRA': 'fr-FR', # Франция + 'GEO': 'ka', # Грузия + 'GRC': 'el-GR', # Греция + 'HRV': 'hr-HR', # Хорватия + 'HUN': 'hu-HU', # Венгрия + 'ISR': 'en-IL', # Израиль + 'ITA': 'it-IT', # Италия + 'JPN': 'ja', # Япония + 'LUX': 'fr-LU', # Люксембург + 'MAR': 'ar-MA', # Марокко + 'MDV': 'dv', # Мальдивы + 'NLD': 'nl-NL', # Нидерланды + 'POL': 'pl', # Польша + 'ROU': 'ro', # Румыния + 'RUS': 'ru-RU', # Россия + 'SVN': 'hu-SI', # Словения + 'USA': 'en-US', # США +} diff --git a/apps/utils/legacy_parser.py b/apps/utils/legacy_parser.py new file mode 100644 index 00000000..7fbbd133 --- /dev/null +++ b/apps/utils/legacy_parser.py @@ -0,0 +1,50 @@ +from pprint import pprint + +import yaml + + +def parse_legacy_news_content(legacy_content): + clear_str = '!ruby/hash:ActiveSupport::HashWithIndifferentAccess' + content_dict = yaml.safe_load(legacy_content.replace(clear_str, '')) + result = '' + try: + result = content_dict['news_content']['value'] + except KeyError: + pass + return result + + +def parse_legacy_schedule_content(legacy_content): + initial = legacy_content + + s1 = "!ruby/object:ActionController::Parameters" + s2 = "!ruby/hash:ActiveSupport::HashWithIndifferentAccess" + + for _ in (s1, s2): + legacy_content = legacy_content.replace(_, "") + content_dict = yaml.safe_load(legacy_content) + + if s1 not in initial or s2 not in initial: + return yaml.safe_load(legacy_content) + + result = {} + for k, v in content_dict.items(): + try: + if v["parameters"]["afternoon"]: + afternoon = v["parameters"]["afternoon"]["parameters"] + else: + afternoon = None + + if v["parameters"]["morning"]: + morning = v["parameters"]["morning"]["parameters"] + else: + morning = None + + data = {"afternoon": afternoon, "morning": morning} + result.update({k: data}) + except KeyError: + print('--------' * 7) + pprint(initial) + raise KeyError + + return result diff --git a/apps/utils/models.py b/apps/utils/models.py index 03330eb4..0c94d23f 100644 --- a/apps/utils/models.py +++ b/apps/utils/models.py @@ -332,3 +332,6 @@ class GMTokenGenerator(PasswordResetTokenGenerator): used. """ return self.get_fields(user, timestamp) + + +timezone.datetime.now().date().isoformat() \ No newline at end of file diff --git a/apps/utils/serializers.py b/apps/utils/serializers.py index eeff1043..bd7e8b8e 100644 --- a/apps/utils/serializers.py +++ b/apps/utils/serializers.py @@ -4,6 +4,7 @@ from django.core import exceptions from rest_framework import serializers from utils import models from translation.models import Language +from favorites.models import Favorites class EmptySerializer(serializers.Serializer): @@ -72,3 +73,28 @@ class ProjectModelSerializer(serializers.ModelSerializer): """Overrided ModelSerializer.""" serializers.ModelSerializer.serializer_field_mapping[models.TJSONField] = TJSONField + + +class FavoritesCreateSerializer(serializers.ModelSerializer): + """Serializer to favorite object.""" + + class Meta: + """Serializer for model Comment.""" + model = Favorites + fields = [ + 'id', + 'created', + ] + + @property + def request(self): + return self.context.get('request') + + @property + def user(self): + """Get user from request""" + return self.request.user + + @property + def slug(self): + return self.request.parser_context.get('kwargs').get('slug') diff --git a/apps/utils/slug_generator.py b/apps/utils/slug_generator.py new file mode 100644 index 00000000..fa1ec772 --- /dev/null +++ b/apps/utils/slug_generator.py @@ -0,0 +1,15 @@ +from django.utils.text import slugify + + +def generate_unique_slug(klass, text): + """ + return unique slug if origin slug is exist. + eg: `foo-bar` => `foo-bar-1` + """ + origin_slug = slugify(text) + unique_slug = origin_slug + numb = 1 + while klass.objects.filter(slug=unique_slug).exists(): + unique_slug = f'{origin_slug}-{numb}' + numb += 1 + return unique_slug diff --git a/celerybeat-schedule b/celerybeat-schedule index e1a56a15..42475a62 100644 Binary files a/celerybeat-schedule and b/celerybeat-schedule differ diff --git a/docker-compose.mysql.yml b/docker-compose.mysql.yml new file mode 100644 index 00000000..4d4e553e --- /dev/null +++ b/docker-compose.mysql.yml @@ -0,0 +1,141 @@ +version: '3.5' +services: + + # Legacy MySQL DB + mysql_db: + image: mysql:5.7 + ports: + - "3306:3306" + environment: + MYSQL_DATABASE: dev + MYSQL_USER: dev + MYSQL_PASSWORD: octosecret123 + MYSQL_ROOT_PASSWORD: rootPassword + volumes: + - .:/code + - gm-mysql_db:/var/lib/mysql + networks: + - mysql_network + + # PostgreSQL database + db: + build: + context: ./_dockerfiles/db + dockerfile: Dockerfile + hostname: db + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + ports: + - "5436:5432" + volumes: + - gm-db:/var/lib/postgresql/data/ + networks: + - database_network + + elasticsearch: + image: elasticsearch:7.3.1 + volumes: + - gm-esdata:/usr/share/elasticsearch/data + hostname: elasticsearch + ports: + - 9200:9200 + - 9300:9300 + environment: + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - discovery.type=single-node + - xpack.security.enabled=false + networks: + - elasticsearch_network + + # Redis + redis: + image: redis:alpine + networks: + - redis_network + + # Celery + worker: + build: . + command: ./run_celery.sh + environment: + - SETTINGS_CONFIGURATION=local + - DB_NAME=postgres + - DB_USERNAME=postgres + - DB_HOSTNAME=db + - DB_PORT=5432 + - DB_PASSWORD=postgres + volumes: + - .:/code + links: + - db + - redis + networks: + - redis_network + + worker_beat: + build: . + command: ./run_celery_beat.sh + environment: + - SETTINGS_CONFIGURATION=local + - DB_NAME=postgres + - DB_USERNAME=postgres + - DB_HOSTNAME=db + - DB_PORT=5432 + - DB_PASSWORD=postgres + volumes: + - .:/code + links: + - db + - redis + networks: + - redis_network + + # App: G&M + gm_app: + build: . + command: python manage.py runserver 0.0.0.0:8000 + environment: + - SETTINGS_CONFIGURATION=local + - DB_HOSTNAME=db + - DB_PORT=5432 + - DB_NAME=postgres + - DB_USERNAME=postgres + - DB_PASSWORD=postgres + depends_on: + - mysql_db + - db + - redis + - worker + - worker_beat + - elasticsearch + volumes: + - .:/code + - gm-media:/media-data + ports: + - "8000:8000" + networks: + - redis_network + - database_network + - mysql_network + - elasticsearch_network + +volumes: + gm-mysql_db: + name: gm-mysql_db + gm-db: + name: gm-db + gm-media: + name: gm-media + gm-esdata: + +networks: + database_network: + driver: bridge + mysql_network: + driver: bridge + elasticsearch_network: + driver: bridge + redis_network: + driver: bridge diff --git a/fabfile.py b/fabfile.py index 9ad7f871..e8a52f85 100644 --- a/fabfile.py +++ b/fabfile.py @@ -53,12 +53,14 @@ def collectstatic(): def deploy(branch=None): - fetch() - install_requirements() - migrate() - collectstatic() - touch() - kill_celery() + role = env.roles[0] + if env.roledefs[role]['branch'] != 'develop': + fetch() + install_requirements() + migrate() + collectstatic() + touch() + kill_celery() def rev(): diff --git a/project/settings/base.py b/project/settings/base.py index 85274993..d2cf7b49 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -74,6 +74,7 @@ PROJECT_APPS = [ 'comment.apps.CommentConfig', 'favorites.apps.FavoritesConfig', 'rating.apps.RatingConfig', + 'transfer.apps.TransferConfig', 'tag.apps.TagConfig', 'product.apps.ProductConfig', ] @@ -156,6 +157,16 @@ DATABASES = { 'HOST': os.environ.get('DB_HOSTNAME'), 'PORT': os.environ.get('DB_PORT'), }, + 'legacy': { + 'ENGINE': 'django.db.backends.mysql', + 'HOST': '172.17.0.1', + # 'HOST': '172.23.0.1', + # 'HOST': 'mysql_db', + 'PORT': 3306, + 'NAME': 'dev', + 'USER': 'dev', + 'PASSWORD': 'octosecret123' + } } @@ -318,7 +329,8 @@ REDOC_SETTINGS = { # RabbitMQ # BROKER_URL = 'amqp://rabbitmq:5672' # Redis -BROKER_URL = 'redis://localhost:6379/1' +# BROKER_URL = 'redis://localhost:6379/1' +BROKER_URL = 'redis://redis:6379/1' CELERY_RESULT_BACKEND = BROKER_URL CELERY_BROKER_URL = BROKER_URL CELERY_ACCEPT_CONTENT = ['application/json'] @@ -473,3 +485,6 @@ STATICFILES_DIRS = ( # MEDIA MEDIA_LOCATION = 'media' + +PHONENUMBER_DB_FORMAT = 'NATIONAL' +PHONENUMBER_DEFAULT_REGION = "FR" diff --git a/project/settings/development.py b/project/settings/development.py index 9f950abf..a9cfe5be 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -20,6 +20,7 @@ DOMAIN_URI = 'gm.id-east.ru' ELASTICSEARCH_DSL = { 'default': { 'hosts': 'localhost:9200' + # 'hosts': 'elasticsearch:9200' } } @@ -38,3 +39,25 @@ sentry_sdk.init( # TMP ( TODO remove it later) # Временный хардкод для демонстрации 4 ноября, потом удалить! HARDCODED_INTERNATIONAL_NEWS_IDS = [8, 9, 10, 11, 15, 17] + +# Database +# https://docs.djangoproject.com/en/2.2/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.contrib.gis.db.backends.postgis', + 'NAME': os.environ.get('DB_NAME'), + 'USER': os.environ.get('DB_USERNAME'), + 'PASSWORD': os.environ.get('DB_PASSWORD'), + 'HOST': os.environ.get('DB_HOSTNAME'), + 'PORT': os.environ.get('DB_PORT'), + }, + 'legacy': { + 'ENGINE': 'django.db.backends.mysql', + 'HOST': os.environ.get('MYSQL_HOSTNAME'), + 'PORT': os.environ.get('MYSQL_PORT'), + 'NAME': os.environ.get('MYSQL_DATABASE'), + 'USER': os.environ.get('MYSQL_USER'), + 'PASSWORD': os.environ.get('MYSQL_PASSWORD') + } +} \ No newline at end of file diff --git a/requirements/base.txt b/requirements/base.txt index c95055de..3a45dedc 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -41,7 +41,11 @@ django-storages==1.7.2 sorl-thumbnail==12.5.0 + +mysqlclient==1.4.4 +PyYAML==5.1.2 + # temp solution redis==3.2.0 amqp>=2.4.0 -celery==4.3.0rc2 \ No newline at end of file +celery==4.3.0rc2